diff --git a/app/helpers/imports_helper.rb b/app/helpers/imports_helper.rb index 61948a7b3..30f986bce 100644 --- a/app/helpers/imports_helper.rb +++ b/app/helpers/imports_helper.rb @@ -31,21 +31,30 @@ def import_col_label(key) def dry_run_resource(key) map = { - transactions: DryRunResource.new(label: "Transactions", icon: "credit-card", text_class: "text-cyan-500", bg_class: "bg-cyan-500/5"), - accounts: DryRunResource.new(label: "Accounts", icon: "layers", text_class: "text-orange-500", bg_class: "bg-orange-500/5"), - categories: DryRunResource.new(label: "Categories", icon: "shapes", text_class: "text-blue-500", bg_class: "bg-blue-500/5"), - tags: DryRunResource.new(label: "Tags", icon: "tags", text_class: "text-violet-500", bg_class: "bg-violet-500/5"), - rules: DryRunResource.new(label: "Rules", icon: "workflow", text_class: "text-green-500", bg_class: "bg-green-500/5"), - merchants: DryRunResource.new(label: "Merchants", icon: "store", text_class: "text-amber-500", bg_class: "bg-amber-500/5"), - trades: DryRunResource.new(label: "Trades", icon: "arrow-left-right", text_class: "text-emerald-500", bg_class: "bg-emerald-500/5"), - valuations: DryRunResource.new(label: "Valuations", icon: "trending-up", text_class: "text-pink-500", bg_class: "bg-pink-500/5"), - budgets: DryRunResource.new(label: "Budgets", icon: "wallet", text_class: "text-indigo-500", bg_class: "bg-indigo-500/5"), - budget_categories: DryRunResource.new(label: "Budget Categories", icon: "pie-chart", text_class: "text-teal-500", bg_class: "bg-teal-500/5") + transactions: DryRunResource.new(label: t("imports.dry_run_resources.transactions"), icon: "credit-card", text_class: "text-info", bg_class: "bg-info/10"), + balances: DryRunResource.new(label: t("imports.dry_run_resources.balances"), icon: "line-chart", text_class: "text-secondary", bg_class: "bg-container-inset"), + accounts: DryRunResource.new(label: t("imports.dry_run_resources.accounts"), icon: "layers", text_class: "text-warning", bg_class: "bg-warning/10"), + categories: DryRunResource.new(label: t("imports.dry_run_resources.categories"), icon: "shapes", text_class: "text-info", bg_class: "bg-info/10"), + tags: DryRunResource.new(label: t("imports.dry_run_resources.tags"), icon: "tags", text_class: "text-info", bg_class: "bg-info/10"), + rules: DryRunResource.new(label: t("imports.dry_run_resources.rules"), icon: "workflow", text_class: "text-success", bg_class: "bg-success/10"), + merchants: DryRunResource.new(label: t("imports.dry_run_resources.merchants"), icon: "store", text_class: "text-warning", bg_class: "bg-warning/10"), + recurring_transactions: DryRunResource.new(label: t("imports.dry_run_resources.recurring_transactions"), icon: "repeat-2", text_class: "text-secondary", bg_class: "bg-container-inset"), + transfers: DryRunResource.new(label: t("imports.dry_run_resources.transfers"), icon: "repeat", text_class: "text-secondary", bg_class: "bg-container-inset"), + rejected_transfers: DryRunResource.new(label: t("imports.dry_run_resources.rejected_transfers"), icon: "ban", text_class: "text-destructive", bg_class: "bg-destructive/10"), + trades: DryRunResource.new(label: t("imports.dry_run_resources.trades"), icon: "arrow-left-right", text_class: "text-success", bg_class: "bg-success/10"), + holdings: DryRunResource.new(label: t("imports.dry_run_resources.holdings"), icon: "briefcase-business", text_class: "text-secondary", bg_class: "bg-container-inset"), + valuations: DryRunResource.new(label: t("imports.dry_run_resources.valuations"), icon: "trending-up", text_class: "text-info", bg_class: "bg-info/10"), + budgets: DryRunResource.new(label: t("imports.dry_run_resources.budgets"), icon: "wallet", text_class: "text-info", bg_class: "bg-info/10"), + budget_categories: DryRunResource.new(label: t("imports.dry_run_resources.budget_categories"), icon: "pie-chart", text_class: "text-success", bg_class: "bg-success/10") } map[key] end + def import_verification_view(import) + ImportVerificationView.new(import.verification_payload) + end + def permitted_import_configuration_path(import) if permitted_import_types.include?(import.type.underscore) "import/configurations/#{import.type.underscore}" @@ -75,4 +84,49 @@ def permitted_import_types end DryRunResource = Struct.new(:label, :icon, :text_class, :bg_class, keyword_init: true) + + ImportVerificationView = Struct.new(:payload) do + def status + readback.fetch("status", "not_verified").to_s + end + + def checked_total + checked_counts.values.sum(&:to_i) + end + + def checked_counts + hash_value(readback["checked_counts"]) + end + + def mismatches + hash_value(readback["mismatches"]) + end + + def mismatches_count + mismatches.size + end + + def mismatches_preview + mismatches.first(3) + end + + def mismatches? + mismatches.any? + end + + private + def readback + hash_value(payload_hash["readback"]) + end + + def payload_hash + hash_value(payload) + end + + def hash_value(value) + return {} unless value.respond_to?(:to_h) + + value.to_h.deep_stringify_keys + end + end end diff --git a/app/models/sure_import.rb b/app/models/sure_import.rb index 23815437a..0fab88a27 100644 --- a/app/models/sure_import.rb +++ b/app/models/sure_import.rb @@ -2,16 +2,22 @@ class SureImport < Import MAX_NDJSON_SIZE = 10.megabytes IMPORTABLE_NDJSON_TYPES = { "Account" => :accounts, + "Balance" => :balances, "Category" => :categories, "Tag" => :tags, "Merchant" => :merchants, + "RecurringTransaction" => :recurring_transactions, "Transaction" => :transactions, + "Transfer" => :transfers, + "RejectedTransfer" => :rejected_transfers, "Trade" => :trades, + "Holding" => :holdings, "Valuation" => :valuations, "Budget" => :budgets, "BudgetCategory" => :budget_categories, "Rule" => :rules }.freeze + VERIFICATION_STATUSES = %w[not_verified matched mismatch failed reverted].freeze ALLOWED_NDJSON_CONTENT_TYPES = %w[ application/x-ndjson application/ndjson @@ -59,6 +65,14 @@ def dry_run_totals_from_line_type_counts(counts) end end + def expected_record_counts_from_ndjson(content) + expected_record_counts_from_line_type_counts(ndjson_line_type_counts(content)) + end + + def expected_record_counts_from_line_type_counts(counts) + dry_run_totals_from_line_type_counts(counts).transform_keys(&:to_s) + end + def importable_ndjson_types IMPORTABLE_NDJSON_TYPES.keys end @@ -105,11 +119,18 @@ def dry_run end def import! + sync_ndjson_counts! + before_counts = readback_count_snapshot importer = Family::DataImporter.new(family, ndjson_blob_string) result = importer.import! result[:accounts].each { |account| accounts << account } result[:entries].each { |entry| entries << entry } + + record_readback_verification!(before_counts:) + rescue => error + record_failed_readback_verification!(before_counts:, error:) + raise end def uploaded? @@ -146,12 +167,133 @@ def max_row_count def sync_ndjson_rows_count! return unless ndjson_file.attached? - total = self.class.ndjson_line_type_counts(ndjson_blob_string).values.sum - update_column(:rows_count, total) + sync_ndjson_counts! + end + + def verification_payload + { + expected_record_counts: normalized_expected_record_counts, + readback: normalized_readback_verification + } + end + + def verification_status + status = normalized_readback_verification["status"] + status.in?(VERIFICATION_STATUSES) ? status : "not_verified" + end + + def reset_readback_verification! + update_columns( + readback_verification: { + "status" => "reverted", + "checked_at" => Time.current.iso8601 + }, + updated_at: Time.current + ) + end + + def revert + super + reset_readback_verification! if pending? end private + def sync_ndjson_counts! + line_counts = self.class.ndjson_line_type_counts(ndjson_blob_string) + + update_columns( + rows_count: line_counts.values.sum, + expected_record_counts: self.class.expected_record_counts_from_line_type_counts(line_counts), + readback_verification: {}, + updated_at: Time.current + ) + end + + def record_readback_verification!(before_counts:) + update_columns( + readback_verification: build_readback_verification(before_counts:, status_for_mismatch: "mismatch"), + updated_at: Time.current + ) + end + + def record_failed_readback_verification!(before_counts:, error:) + return unless before_counts + + update_columns( + readback_verification: build_readback_verification(before_counts:, status_for_mismatch: "failed").merge( + "status" => "failed", + "error" => error.message + ), + updated_at: Time.current + ) + rescue => verification_error + Rails.logger.warn("Failed to record Sure import readback verification for import #{id}: #{verification_error.message}") + end + + def build_readback_verification(before_counts:, status_for_mismatch:) + after_counts = readback_count_snapshot + actual_delta_counts = delta_counts(before_counts, after_counts) + expected_counts = normalized_expected_record_counts + checked_counts = (actual_delta_counts.keys | expected_counts.keys).index_with do |key| + expected_counts.fetch(key, 0).to_i + end + mismatches = checked_counts.each_with_object({}) do |(key, expected_count), result| + actual_count = actual_delta_counts.fetch(key, 0) + next if actual_count == expected_count.to_i + + result[key] = { + "expected" => expected_count.to_i, + "actual" => actual_count + } + end + + { + "status" => mismatches.empty? ? "matched" : status_for_mismatch, + "checked_at" => Time.current.iso8601, + "expected_record_counts" => expected_counts, + "before_counts" => before_counts, + "after_counts" => after_counts, + "actual_delta_counts" => actual_delta_counts, + "checked_counts" => checked_counts, + "mismatches" => mismatches + } + end + + def readback_count_snapshot + { + accounts: family.accounts.count, + balances: Balance.joins(:account).where(accounts: { family_id: family.id }).count, + categories: family.categories.count, + tags: family.tags.count, + merchants: family.merchants.count, + recurring_transactions: family.recurring_transactions.count, + transactions: family.entries.where(entryable_type: "Transaction").count, + transfers: Transfer.joins(inflow_transaction: { entry: :account }).where(accounts: { family_id: family.id }).count, + rejected_transfers: RejectedTransfer.joins(inflow_transaction: { entry: :account }).where(accounts: { family_id: family.id }).count, + trades: family.entries.where(entryable_type: "Trade").count, + holdings: family.holdings.count, + valuations: family.entries.where(entryable_type: "Valuation").count, + budgets: family.budgets.count, + budget_categories: family.budget_categories.count, + rules: family.rules.count + }.transform_keys(&:to_s).transform_values(&:to_i) + end + + def delta_counts(before_counts, after_counts) + after_counts.each_with_object({}) do |(key, after_count), result| + result[key] = after_count.to_i - before_counts.fetch(key, 0).to_i + end + end + + def normalized_expected_record_counts + (expected_record_counts || {}).to_h.transform_keys(&:to_s).transform_values(&:to_i) + end + + def normalized_readback_verification + (readback_verification || {}).to_h.deep_stringify_keys + end + def ndjson_blob_string blob_id = ndjson_file.blob&.id diff --git a/app/views/api/v1/imports/show.json.jbuilder b/app/views/api/v1/imports/show.json.jbuilder index 2ce5f0e52..fd8526428 100644 --- a/app/views/api/v1/imports/show.json.jbuilder +++ b/app/views/api/v1/imports/show.json.jbuilder @@ -44,6 +44,8 @@ json.data do json.unassigned_mappings_count mapping_counts[:unassigned_mappings_count] end + json.verification @import.verification_payload if @import.is_a?(SureImport) + # Only show a subset of rows for preview if needed, or link to a separate rows endpoint # json.sample_rows @import.rows.limit(5) end diff --git a/app/views/imports/_success.html.erb b/app/views/imports/_success.html.erb index c3bc2d1eb..329977a91 100644 --- a/app/views/imports/_success.html.erb +++ b/app/views/imports/_success.html.erb @@ -2,7 +2,7 @@
-
+
<%= icon "check", color: "success" %>
@@ -11,6 +11,39 @@

<%= t(".description") %>

+ <% if import.is_a?(SureImport) %> + <% verification = import_verification_view(import) %> + +
+
+

<%= t(".verification.title") %>

+ <%= t(".verification.status.#{verification.status}", default: verification.status.humanize) %> +
+ +
+
+

<%= t(".verification.checked") %>

+

<%= number_with_delimiter(verification.checked_total) %>

+
+
+

<%= t(".verification.mismatches") %>

+

<%= number_with_delimiter(verification.mismatches_count) %>

+
+
+ + <% if verification.mismatches? %> +
+ <% verification.mismatches_preview.each do |key, counts| %> +
+ <%= key.humanize %> + <%= number_with_delimiter(counts["actual"]) %> / <%= number_with_delimiter(counts["expected"]) %> +
+ <% end %> +
+ <% end %> +
+ <% end %> + <%= render DS::Link.new( text: t(".back_to_dashboard"), variant: "primary", diff --git a/config/locales/views/imports/en.yml b/config/locales/views/imports/en.yml index f6d5c1523..270b7d2f6 100644 --- a/config/locales/views/imports/en.yml +++ b/config/locales/views/imports/en.yml @@ -217,6 +217,22 @@ en: account: "Account" category: "Category" tag: "Tag" + dry_run_resources: + transactions: "Transactions" + balances: "Balances" + accounts: "Accounts" + categories: "Categories" + tags: "Tags" + rules: "Rules" + merchants: "Merchants" + recurring_transactions: "Recurring Transactions" + transfers: "Transfers" + rejected_transfers: "Rejected Transfers" + trades: "Trades" + holdings: "Holdings" + valuations: "Valuations" + budgets: "Budgets" + budget_categories: "Budget Categories" column_labels: date: "Date" amount: "Amount" @@ -255,6 +271,16 @@ en: title: Import successful description: Your imported data has been successfully added to the app and is now ready for use. back_to_dashboard: Back to dashboard + verification: + title: Readback verification + checked: Checked + mismatches: Mismatches + status: + not_verified: Not verified + matched: Matched + mismatch: Mismatch + failed: Failed + reverted: Reverted importing: title: Import in progress description: "Your import is in progress. Check the imports menu for status updates or click 'Check Status' to refresh the page for updates. Feel free to continue using the app." diff --git a/db/migrate/20260519100000_add_sure_import_verification_to_imports.rb b/db/migrate/20260519100000_add_sure_import_verification_to_imports.rb new file mode 100644 index 000000000..d73a24fd5 --- /dev/null +++ b/db/migrate/20260519100000_add_sure_import_verification_to_imports.rb @@ -0,0 +1,6 @@ +class AddSureImportVerificationToImports < ActiveRecord::Migration[7.2] + def change + add_column :imports, :expected_record_counts, :jsonb, null: false, default: {} + add_column :imports, :readback_verification, :jsonb, null: false, default: {} + end +end diff --git a/db/schema.rb b/db/schema.rb index f2e1a0cc7..c7fcf54ae 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.2].define(version: 2026_05_19_092118) do +ActiveRecord::Schema[7.2].define(version: 2026_05_19_100000) do # These are extensions that must be enabled in order to support this database enable_extension "pgcrypto" enable_extension "plpgsql" @@ -921,6 +921,8 @@ t.text "ai_summary" t.string "document_type" t.jsonb "extracted_data" + t.jsonb "expected_record_counts", default: {}, null: false + t.jsonb "readback_verification", default: {}, null: false t.index ["family_id"], name: "index_imports_on_family_id" end diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index a26146ef5..456e7ffbf 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -1733,6 +1733,72 @@ components: unassigned_mappings_count: type: integer minimum: 0 + ImportVerificationReadback: + type: object + description: SureImport only. Expected NDJSON counts compared to family-scoped + database readback after publish. + properties: + status: + type: string + enum: + - not_verified + - matched + - mismatch + - failed + - reverted + checked_at: + type: string + format: date-time + nullable: true + expected_record_counts: + type: object + additionalProperties: + type: integer + before_counts: + type: object + additionalProperties: + type: integer + after_counts: + type: object + additionalProperties: + type: integer + actual_delta_counts: + type: object + additionalProperties: + type: integer + checked_counts: + type: object + additionalProperties: + type: integer + mismatches: + type: object + additionalProperties: + type: object + required: + - expected + - actual + properties: + expected: + type: integer + actual: + type: integer + error: + type: string + nullable: true + ImportVerification: + type: object + description: SureImport only. Captured at upload and completed after import + publish. + required: + - expected_record_counts + - readback + properties: + expected_record_counts: + type: object + additionalProperties: + type: integer + readback: + "$ref": "#/components/schemas/ImportVerificationReadback" ImportPreflightContent: type: object required: @@ -1975,6 +2041,8 @@ components: "$ref": "#/components/schemas/ImportConfiguration" stats: "$ref": "#/components/schemas/ImportStats" + verification: + "$ref": "#/components/schemas/ImportVerification" ImportCollection: type: object required: @@ -4753,7 +4821,7 @@ paths: get: summary: Retrieve an import description: Retrieve detailed information about a specific import, including - configuration and row statistics. + configuration, row statistics, and SureImport readback verification when available. tags: - Imports security: diff --git a/spec/requests/api/v1/imports_spec.rb b/spec/requests/api/v1/imports_spec.rb index cd48a35ae..7d6691612 100644 --- a/spec/requests/api/v1/imports_spec.rb +++ b/spec/requests/api/v1/imports_spec.rb @@ -286,7 +286,7 @@ parameter name: :id, in: :path, type: :string, required: true, description: 'Import ID' get 'Retrieve an import' do - description 'Retrieve detailed information about a specific import, including configuration and row statistics.' + description 'Retrieve detailed information about a specific import, including configuration, row statistics, and SureImport readback verification when available.' tags 'Imports' security [ { apiKeyAuth: [] } ] produces 'application/json' diff --git a/spec/swagger_helper.rb b/spec/swagger_helper.rb index ab8ee9783..0372ce03a 100644 --- a/spec/swagger_helper.rb +++ b/spec/swagger_helper.rb @@ -947,6 +947,58 @@ unassigned_mappings_count: { type: :integer, minimum: 0 } } }, + ImportVerificationReadback: { + type: :object, + description: 'SureImport only. Expected NDJSON counts compared to family-scoped database readback after publish.', + properties: { + status: { type: :string, enum: %w[not_verified matched mismatch failed reverted] }, + checked_at: { type: :string, format: :'date-time', nullable: true }, + expected_record_counts: { + type: :object, + additionalProperties: { type: :integer } + }, + before_counts: { + type: :object, + additionalProperties: { type: :integer } + }, + after_counts: { + type: :object, + additionalProperties: { type: :integer } + }, + actual_delta_counts: { + type: :object, + additionalProperties: { type: :integer } + }, + checked_counts: { + type: :object, + additionalProperties: { type: :integer } + }, + mismatches: { + type: :object, + additionalProperties: { + type: :object, + required: %w[expected actual], + properties: { + expected: { type: :integer }, + actual: { type: :integer } + } + } + }, + error: { type: :string, nullable: true } + } + }, + ImportVerification: { + type: :object, + description: 'SureImport only. Captured at upload and completed after import publish.', + required: %w[expected_record_counts readback], + properties: { + expected_record_counts: { + type: :object, + additionalProperties: { type: :integer } + }, + readback: { '$ref' => '#/components/schemas/ImportVerificationReadback' } + } + }, ImportPreflightContent: { type: :object, required: %w[filename content_type byte_size], @@ -1086,7 +1138,8 @@ error: { type: :string, nullable: true }, status_detail: { '$ref' => '#/components/schemas/ImportStatusDetail' }, configuration: { '$ref' => '#/components/schemas/ImportConfiguration' }, - stats: { '$ref' => '#/components/schemas/ImportStats' } + stats: { '$ref' => '#/components/schemas/ImportStats' }, + verification: { '$ref' => '#/components/schemas/ImportVerification' } } }, ImportCollection: { diff --git a/test/controllers/api/v1/imports_controller_test.rb b/test/controllers/api/v1/imports_controller_test.rb index dcf83ab44..41d6c2e43 100644 --- a/test/controllers/api/v1/imports_controller_test.rb +++ b/test/controllers/api/v1/imports_controller_test.rb @@ -120,6 +120,47 @@ class Api::V1::ImportsControllerTest < ActionDispatch::IntegrationTest json_response["data"]["stats"]["unassigned_mappings_count"] end + test "should show Sure import verification" do + sure_import = @family.imports.create!(type: "SureImport") + sure_import.ndjson_file.attach( + io: StringIO.new(build_ndjson([ + { type: "Account", data: { + id: "account-1", + name: "API Verified Checking", + balance: "100.00", + currency: "USD", + accountable_type: "Depository" + } }, + { type: "Valuation", data: { + id: "valuation-1", + account_id: "account-1", + date: "2024-01-14", + amount: "100.00", + currency: "USD", + kind: "opening_anchor" + } } + ])), + filename: "sure.ndjson", + content_type: "application/x-ndjson" + ) + sure_import.sync_ndjson_rows_count! + sure_import.publish + + get api_v1_import_url(sure_import), headers: api_headers(@api_key) + + assert_response :success + json_response = JSON.parse(response.body) + verification = json_response.dig("data", "verification") + + assert_equal 1, verification.dig("expected_record_counts", "accounts") + assert_equal 1, verification.dig("expected_record_counts", "valuations") + assert_equal "matched", verification.dig("readback", "status") + assert_equal 1, verification.dig("readback", "actual_delta_counts", "accounts") + assert_equal 1, verification.dig("readback", "actual_delta_counts", "valuations") + assert_equal 0, verification.dig("readback", "checked_counts", "balances") + assert_empty verification.dig("readback", "mismatches") + end + test "should list sanitized import row diagnostics" do get rows_api_v1_import_url(@diagnostic_import), headers: api_headers(@read_only_api_key) @@ -1183,6 +1224,10 @@ class Api::V1::ImportsControllerTest < ActionDispatch::IntegrationTest private + def build_ndjson(records) + records.map(&:to_json).join("\n") + end + def api_headers(api_key) { "X-Api-Key" => api_key.plain_key } end diff --git a/test/helpers/imports_helper_test.rb b/test/helpers/imports_helper_test.rb new file mode 100644 index 000000000..5640f891b --- /dev/null +++ b/test/helpers/imports_helper_test.rb @@ -0,0 +1,77 @@ +require "test_helper" +require "ostruct" + +class ImportsHelperTest < ActionView::TestCase + test "dry run resource labels come from locale keys" do + %i[ + transactions balances accounts categories tags rules merchants + recurring_transactions transfers rejected_transfers trades holdings + valuations budgets budget_categories + ].each do |key| + assert_equal I18n.t("imports.dry_run_resources.#{key}"), dry_run_resource(key).label + end + + balances = dry_run_resource(:balances) + assert_equal "line-chart", balances.icon + assert_equal "text-secondary", balances.text_class + assert_equal "bg-container-inset", balances.bg_class + + resources = %i[ + transactions balances accounts categories tags rules merchants + recurring_transactions transfers rejected_transfers trades holdings + valuations budgets budget_categories + ].map { |key| dry_run_resource(key) } + resources.each do |resource| + refute_match(/\b(?:text|bg)-[a-z]+-\d{2,3}/, [ resource.text_class, resource.bg_class ].join(" ")) + end + end + + test "import verification view handles missing readback payload" do + import = OpenStruct.new(verification_payload: {}) + + verification = import_verification_view(import) + + assert_equal "not_verified", verification.status + assert_equal 0, verification.checked_total + assert_equal 0, verification.mismatches_count + assert_empty verification.mismatches_preview + refute verification.mismatches? + end + + test "import verification view handles nil readback payload" do + import = OpenStruct.new(verification_payload: { readback: nil }) + + verification = import_verification_view(import) + + assert_equal "not_verified", verification.status + assert_equal 0, verification.checked_total + assert_equal 0, verification.mismatches_count + assert_empty verification.mismatches_preview + refute verification.mismatches? + end + + test "import verification view shapes readback counts and mismatch preview" do + import = OpenStruct.new( + verification_payload: { + readback: { + status: "mismatch", + checked_counts: { accounts: 1, transactions: "2" }, + mismatches: { + accounts: { expected: 1, actual: 0 }, + transactions: { expected: 2, actual: 1 }, + categories: { expected: 3, actual: 2 }, + tags: { expected: 4, actual: 3 } + } + } + } + ) + + verification = import_verification_view(import) + + assert_equal "mismatch", verification.status + assert_equal 3, verification.checked_total + assert_equal 4, verification.mismatches_count + assert_equal %w[accounts transactions categories], verification.mismatches_preview.map(&:first) + assert verification.mismatches? + end +end diff --git a/test/models/sure_import_test.rb b/test/models/sure_import_test.rb index 9345dea60..ff99bb6fd 100644 --- a/test/models/sure_import_test.rb +++ b/test/models/sure_import_test.rb @@ -164,6 +164,52 @@ class SureImportTest < ActiveSupport::TestCase assert_equal 3, @import.rows_count end + test "sync_ndjson_rows_count! persists expected record counts" do + attach_ndjson(build_ndjson([ + { type: "Account", data: { id: "account-1" } }, + { type: "Balance", data: { id: "balance-1" } }, + { type: "Transaction", data: { id: "transaction-1" } }, + { type: "UnknownType", data: { id: "unknown-1" } } + ])) + + @import.reload + + assert_equal 4, @import.rows_count + assert_equal 1, @import.expected_record_counts["accounts"] + assert_equal 1, @import.expected_record_counts["balances"] + assert_equal 1, @import.expected_record_counts["transactions"] + assert_not @import.expected_record_counts.key?("unknown_type") + assert_equal({}, @import.readback_verification) + end + + test "import resyncs expected counts from current attachment" do + attach_ndjson(build_ndjson([ + { type: "Account", data: { id: "stale-account" } } + ])) + @import.ndjson_file.attach( + io: StringIO.new(build_ndjson([ + { type: "Category", data: { + id: "current-category", + name: "Current Category", + color: "#407706", + classification: "expense", + lucide_icon: "shapes" + } } + ])), + filename: "current.ndjson", + content_type: "application/x-ndjson" + ) + + @import.import! + @import.reload + + assert_equal 1, @import.rows_count + assert_equal 0, @import.expected_record_counts["accounts"] + assert_equal 1, @import.expected_record_counts["categories"] + assert_equal 1, @import.readback_verification.dig("expected_record_counts", "categories") + assert_equal "matched", @import.readback_verification["status"] + end + test "publishes import successfully" do attach_ndjson(build_ndjson([ { type: "Account", data: { @@ -190,6 +236,163 @@ class SureImportTest < ActiveSupport::TestCase assert_equal "Depository", account.accountable_type end + test "publish records matched readback verification from family-scoped deltas" do + other_family = Family.create!(name: "Other Family", currency: "USD", locale: "en", date_format: "%m-%d-%Y") + other_family.accounts.create!( + name: "Other Checking", + balance: 100, + currency: "USD", + accountable: Depository.new + ) + + attach_ndjson(importable_history_ndjson) + + @import.publish + @import.reload + + verification = @import.readback_verification + + assert_equal "complete", @import.status + assert_equal "matched", verification["status"] + assert_equal 1, verification.dig("expected_record_counts", "accounts") + assert_equal 1, verification.dig("expected_record_counts", "categories") + assert_equal 1, verification.dig("expected_record_counts", "tags") + assert_equal 1, verification.dig("expected_record_counts", "merchants") + assert_equal 1, verification.dig("expected_record_counts", "transactions") + assert_equal 1, verification.dig("expected_record_counts", "valuations") + assert_equal 1, verification.dig("actual_delta_counts", "accounts") + assert_equal 1, verification.dig("actual_delta_counts", "categories") + assert_equal 1, verification.dig("actual_delta_counts", "tags") + assert_equal 1, verification.dig("actual_delta_counts", "merchants") + assert_equal 1, verification.dig("actual_delta_counts", "transactions") + assert_equal 1, verification.dig("actual_delta_counts", "valuations") + assert_equal 0, verification.dig("checked_counts", "balances") + assert_empty verification["mismatches"] + assert_equal 1, other_family.accounts.count + end + + test "publish verifies expected zero record types against unexpected readback deltas" do + attach_ndjson(build_ndjson([ + { type: "Account", data: { + id: "account-1", + name: "Implicit Opening Anchor", + balance: "100.00", + currency: "USD", + accountable_type: "Depository", + accountable: { subtype: "checking" } + } } + ])) + + @import.publish + @import.reload + + verification = @import.readback_verification + + assert_equal "complete", @import.status + assert_equal "mismatch", verification["status"] + assert_equal 0, verification.dig("expected_record_counts", "valuations") + assert_equal 0, verification.dig("checked_counts", "valuations") + assert_equal 1, verification.dig("actual_delta_counts", "valuations") + assert_equal({ "expected" => 0, "actual" => 1 }, verification.dig("mismatches", "valuations")) + end + + test "publish records mismatch when expected rows are skipped by readback" do + attach_ndjson(build_ndjson([ + { type: "Transaction", data: { + id: "transaction-1", + account_id: "missing-account", + date: "2024-01-15", + amount: "12.34", + name: "Skipped transaction", + currency: "USD" + } } + ])) + + @import.publish + @import.reload + + assert_equal "complete", @import.status + assert_equal "mismatch", @import.readback_verification["status"] + assert_equal({ "expected" => 1, "actual" => 0 }, @import.readback_verification.dig("mismatches", "transactions")) + end + + test "failed publish records failed verification without partial mutation" do + attach_ndjson(build_ndjson([ + { type: "Account", data: { + id: "account-1", + name: "Rollback Account", + balance: "100.00", + currency: "USD", + accountable_type: "Depository" + } }, + { type: "Transaction", data: { + id: "transaction-1", + account_id: "account-1", + date: "not-a-date", + amount: "12.34", + name: "Bad date", + currency: "USD" + } } + ])) + + initial_account_count = @family.accounts.count + initial_transaction_count = @family.entries.where(entryable_type: "Transaction").count + + @import.publish + @import.reload + + assert_equal "failed", @import.status + assert_equal initial_account_count, @family.accounts.count + assert_equal initial_transaction_count, @family.entries.where(entryable_type: "Transaction").count + assert_equal "failed", @import.readback_verification["status"] + assert_equal 0, @import.readback_verification.dig("actual_delta_counts", "accounts") + assert_equal 0, @import.readback_verification.dig("actual_delta_counts", "transactions") + end + + test "failed publish keeps original error when failed verification cannot be recorded" do + before_counts = @import.send(:readback_count_snapshot) + original_error = StandardError.new("original import failure") + logged_messages = [] + + Rails.logger.stubs(:warn).with do |message| + logged_messages << message unless logged_messages.include?(message) + true + end + @import.stubs(:update_columns).raises(StandardError, "verification write failed") + + @import.send(:record_failed_readback_verification!, before_counts:, error: original_error) + + assert_match(/Failed to record Sure import readback verification/, logged_messages.first) + assert_match(/verification write failed/, logged_messages.first) + end + + test "revert marks Sure readback verification as reverted" do + attach_ndjson(importable_history_ndjson) + + @import.publish + assert_equal "matched", @import.reload.verification_status + + @import.revert + + assert_equal "pending", @import.status + assert_equal "reverted", @import.verification_status + end + + test "revert failure leaves existing Sure readback verification untouched" do + attach_ndjson(importable_history_ndjson) + + @import.publish + verification = @import.reload.readback_verification + + @import.stub(:entries, -> { raise StandardError, "revert failed before pending" }) do + @import.revert + end + + assert_equal "revert_failed", @import.status + assert_equal verification, @import.readback_verification + assert_equal "matched", @import.verification_status + end + test "import tracks created accounts for revert" do attach_ndjson(build_ndjson([ { type: "Account", data: { @@ -239,4 +442,53 @@ def attach_ndjson(ndjson) def build_ndjson(records) records.map(&:to_json).join("\n") end + + def importable_history_ndjson + build_ndjson([ + { type: "Account", data: { + id: "account-1", + name: "Verified Checking", + balance: "1000.00", + currency: "USD", + accountable_type: "Depository", + accountable: { subtype: "checking" } + } }, + { type: "Valuation", data: { + id: "valuation-1", + account_id: "account-1", + date: "2024-01-14", + amount: "1000.00", + currency: "USD", + kind: "opening_anchor" + } }, + { type: "Category", data: { + id: "category-1", + name: "Verified Category", + color: "#407706", + classification: "expense", + lucide_icon: "shapes" + } }, + { type: "Tag", data: { + id: "tag-1", + name: "Verified Tag", + color: "#407706" + } }, + { type: "Merchant", data: { + id: "merchant-1", + name: "Verified Merchant", + color: "#407706" + } }, + { type: "Transaction", data: { + id: "transaction-1", + account_id: "account-1", + category_id: "category-1", + merchant_id: "merchant-1", + tag_ids: [ "tag-1" ], + date: "2024-01-15", + amount: "12.34", + name: "Verified transaction", + currency: "USD" + } } + ]) + end end