feat(imports): verify Sure NDJSON import readback

app/helpers/imports_helper.rb

@@ -31,21 +31,30 @@ def import_col_label(key)
 
   def dry_run_resource(key)
     map = {
-      transactions: DryRunResource.new(label: "Transactions", icon: "credit-card", text_class: "text-cyan-500", bg_class: "bg-cyan-500/5"),
-      accounts: DryRunResource.new(label: "Accounts", icon: "layers", text_class: "text-orange-500", bg_class: "bg-orange-500/5"),
-      categories: DryRunResource.new(label: "Categories", icon: "shapes", text_class: "text-blue-500", bg_class: "bg-blue-500/5"),
-      tags: DryRunResource.new(label: "Tags", icon: "tags", text_class: "text-violet-500", bg_class: "bg-violet-500/5"),
-      rules: DryRunResource.new(label: "Rules", icon: "workflow", text_class: "text-green-500", bg_class: "bg-green-500/5"),
-      merchants: DryRunResource.new(label: "Merchants", icon: "store", text_class: "text-amber-500", bg_class: "bg-amber-500/5"),
-      trades: DryRunResource.new(label: "Trades", icon: "arrow-left-right", text_class: "text-emerald-500", bg_class: "bg-emerald-500/5"),
-      valuations: DryRunResource.new(label: "Valuations", icon: "trending-up", text_class: "text-pink-500", bg_class: "bg-pink-500/5"),
-      budgets: DryRunResource.new(label: "Budgets", icon: "wallet", text_class: "text-indigo-500", bg_class: "bg-indigo-500/5"),
-      budget_categories: DryRunResource.new(label: "Budget Categories", icon: "pie-chart", text_class: "text-teal-500", bg_class: "bg-teal-500/5")
+      transactions: DryRunResource.new(label: t("imports.dry_run_resources.transactions"), icon: "credit-card", text_class: "text-cyan-500", bg_class: "bg-cyan-500/5"),
+      balances: DryRunResource.new(label: t("imports.dry_run_resources.balances"), icon: "line-chart", text_class: "text-secondary", bg_class: "bg-container-inset"),
+      accounts: DryRunResource.new(label: t("imports.dry_run_resources.accounts"), icon: "layers", text_class: "text-orange-500", bg_class: "bg-orange-500/5"),
+      categories: DryRunResource.new(label: t("imports.dry_run_resources.categories"), icon: "shapes", text_class: "text-blue-500", bg_class: "bg-blue-500/5"),
+      tags: DryRunResource.new(label: t("imports.dry_run_resources.tags"), icon: "tags", text_class: "text-violet-500", bg_class: "bg-violet-500/5"),
+      rules: DryRunResource.new(label: t("imports.dry_run_resources.rules"), icon: "workflow", text_class: "text-green-500", bg_class: "bg-green-500/5"),
+      merchants: DryRunResource.new(label: t("imports.dry_run_resources.merchants"), icon: "store", text_class: "text-amber-500", bg_class: "bg-amber-500/5"),
+      recurring_transactions: DryRunResource.new(label: t("imports.dry_run_resources.recurring_transactions"), icon: "repeat-2", text_class: "text-secondary", bg_class: "bg-container-inset"),
+      transfers: DryRunResource.new(label: t("imports.dry_run_resources.transfers"), icon: "repeat", text_class: "text-secondary", bg_class: "bg-container-inset"),
+      rejected_transfers: DryRunResource.new(label: t("imports.dry_run_resources.rejected_transfers"), icon: "ban", text_class: "text-secondary", bg_class: "bg-container-inset"),
+      trades: DryRunResource.new(label: t("imports.dry_run_resources.trades"), icon: "arrow-left-right", text_class: "text-emerald-500", bg_class: "bg-emerald-500/5"),
+      holdings: DryRunResource.new(label: t("imports.dry_run_resources.holdings"), icon: "briefcase-business", text_class: "text-secondary", bg_class: "bg-container-inset"),
+      valuations: DryRunResource.new(label: t("imports.dry_run_resources.valuations"), icon: "trending-up", text_class: "text-pink-500", bg_class: "bg-pink-500/5"),
+      budgets: DryRunResource.new(label: t("imports.dry_run_resources.budgets"), icon: "wallet", text_class: "text-indigo-500", bg_class: "bg-indigo-500/5"),
+      budget_categories: DryRunResource.new(label: t("imports.dry_run_resources.budget_categories"), icon: "pie-chart", text_class: "text-teal-500", bg_class: "bg-teal-500/5")
     }
 
     map[key]
   end
 
+  def import_verification_view(import)
+    ImportVerificationView.new(import.verification_payload)
+  end
+
   def permitted_import_configuration_path(import)
     if permitted_import_types.include?(import.type.underscore)
       "import/configurations/#{import.type.underscore}"
@@ -75,4 +84,49 @@ def permitted_import_types
     end
 
     DryRunResource = Struct.new(:label, :icon, :text_class, :bg_class, keyword_init: true)
+
+    ImportVerificationView = Struct.new(:payload) do
+      def status
+        readback.fetch("status", "not_verified").to_s
+      end
+
+      def checked_total
+        checked_counts.values.sum(&:to_i)
+      end
+
+      def checked_counts
+        hash_value(readback["checked_counts"])
+      end
+
+      def mismatches
+        hash_value(readback["mismatches"])
+      end
+
+      def mismatches_count
+        mismatches.size
+      end
+
+      def mismatches_preview
+        mismatches.first(3)
+      end
+
+      def mismatches?
+        mismatches.any?
+      end
+
+      private
+        def readback
+          hash_value(payload_hash["readback"])
+        end
+
+        def payload_hash
+          hash_value(payload)
+        end
+
+        def hash_value(value)
+          return {} unless value.respond_to?(:to_h)
+
+          value.to_h.deep_stringify_keys
+        end
+    end
 end

app/models/sure_import.rb

@@ -2,16 +2,22 @@ class SureImport < Import
   MAX_NDJSON_SIZE = 10.megabytes
   IMPORTABLE_NDJSON_TYPES = {
     "Account" => :accounts,
+    "Balance" => :balances,
     "Category" => :categories,
     "Tag" => :tags,
     "Merchant" => :merchants,
+    "RecurringTransaction" => :recurring_transactions,
     "Transaction" => :transactions,
+    "Transfer" => :transfers,
+    "RejectedTransfer" => :rejected_transfers,
     "Trade" => :trades,
+    "Holding" => :holdings,
     "Valuation" => :valuations,
     "Budget" => :budgets,
     "BudgetCategory" => :budget_categories,
     "Rule" => :rules
   }.freeze
+  VERIFICATION_STATUSES = %w[not_verified matched mismatch failed reverted].freeze
   ALLOWED_NDJSON_CONTENT_TYPES = %w[
     application/x-ndjson
     application/ndjson
@@ -59,6 +65,14 @@ def dry_run_totals_from_line_type_counts(counts)
       end
     end
 
+    def expected_record_counts_from_ndjson(content)
+      expected_record_counts_from_line_type_counts(ndjson_line_type_counts(content))
+    end
+
+    def expected_record_counts_from_line_type_counts(counts)
+      dry_run_totals_from_line_type_counts(counts).transform_keys(&:to_s)
+    end
+
     def importable_ndjson_types
       IMPORTABLE_NDJSON_TYPES.keys
     end
@@ -105,11 +119,18 @@ def dry_run
   end
 
   def import!
+    sync_ndjson_counts!
+    before_counts = readback_count_snapshot
     importer = Family::DataImporter.new(family, ndjson_blob_string)
     result = importer.import!
 
     result[:accounts].each { |account| accounts << account }
     result[:entries].each { |entry| entries << entry }
+
+    record_readback_verification!(before_counts:)
+  rescue => error
+    record_failed_readback_verification!(before_counts:, error:)
+    raise
   end
 
   def uploaded?
@@ -146,12 +167,133 @@ def max_row_count
   def sync_ndjson_rows_count!
     return unless ndjson_file.attached?
 
-    total = self.class.ndjson_line_type_counts(ndjson_blob_string).values.sum
-    update_column(:rows_count, total)
+    sync_ndjson_counts!
+  end
+
+  def verification_payload
+    {
+      expected_record_counts: normalized_expected_record_counts,
+      readback: normalized_readback_verification
+    }
+  end
+
+  def verification_status
+    status = normalized_readback_verification["status"]
+    status.in?(VERIFICATION_STATUSES) ? status : "not_verified"
+  end
+
+  def reset_readback_verification!
+    update_columns(
+      readback_verification: {
+        "status" => "reverted",
+        "checked_at" => Time.current.iso8601
+      },
+      updated_at: Time.current
+    )
+  end
+
+  def revert
+    super
+    reset_readback_verification! if pending?
   end
 
   private
 
+    def sync_ndjson_counts!
+      line_counts = self.class.ndjson_line_type_counts(ndjson_blob_string)
+
+      update_columns(
+        rows_count: line_counts.values.sum,
+        expected_record_counts: self.class.expected_record_counts_from_line_type_counts(line_counts),
+        readback_verification: {},
+        updated_at: Time.current
+      )
+    end
+
+    def record_readback_verification!(before_counts:)
+      update_columns(
+        readback_verification: build_readback_verification(before_counts:, status_for_mismatch: "mismatch"),
+        updated_at: Time.current
+      )
+    end
+
+    def record_failed_readback_verification!(before_counts:, error:)
+      return unless before_counts
+
+      update_columns(
+        readback_verification: build_readback_verification(before_counts:, status_for_mismatch: "failed").merge(
+          "status" => "failed",
+          "error" => error.message
+        ),
+        updated_at: Time.current
+      )
+    rescue => verification_error
+      Rails.logger.warn("Failed to record Sure import readback verification for import #{id}: #{verification_error.message}")
+    end
+
+    def build_readback_verification(before_counts:, status_for_mismatch:)
+      after_counts = readback_count_snapshot
+      actual_delta_counts = delta_counts(before_counts, after_counts)
+      expected_counts = normalized_expected_record_counts
+      checked_counts = (actual_delta_counts.keys | expected_counts.keys).index_with do |key|
+        expected_counts.fetch(key, 0).to_i
+      end
+      mismatches = checked_counts.each_with_object({}) do |(key, expected_count), result|
+        actual_count = actual_delta_counts.fetch(key, 0)
+        next if actual_count == expected_count.to_i
+
+        result[key] = {
+          "expected" => expected_count.to_i,
+          "actual" => actual_count
+        }
+      end
+
+      {
+        "status" => mismatches.empty? ? "matched" : status_for_mismatch,
+        "checked_at" => Time.current.iso8601,
+        "expected_record_counts" => expected_counts,
+        "before_counts" => before_counts,
+        "after_counts" => after_counts,
+        "actual_delta_counts" => actual_delta_counts,
+        "checked_counts" => checked_counts,
+        "mismatches" => mismatches
+      }
+    end
+
+    def readback_count_snapshot
+      {
+        accounts: family.accounts.count,
+        balances: Balance.joins(:account).where(accounts: { family_id: family.id }).count,
+        categories: family.categories.count,
+        tags: family.tags.count,
+        merchants: family.merchants.count,
+        recurring_transactions: family.recurring_transactions.count,
+        transactions: family.entries.where(entryable_type: "Transaction").count,
+        transfers: Transfer.joins(inflow_transaction: { entry: :account }).where(accounts: { family_id: family.id }).count,
+        rejected_transfers: RejectedTransfer.joins(inflow_transaction: { entry: :account }).where(accounts: { family_id: family.id }).count,
+        trades: family.entries.where(entryable_type: "Trade").count,
+        holdings: family.holdings.count,
+        valuations: family.entries.where(entryable_type: "Valuation").count,
+        budgets: family.budgets.count,
+        budget_categories: family.budget_categories.count,
+        rules: family.rules.count
+      }.transform_keys(&:to_s).transform_values(&:to_i)
+    end
+
+    def delta_counts(before_counts, after_counts)
+      after_counts.each_with_object({}) do |(key, after_count), result|
+        result[key] = after_count.to_i - before_counts.fetch(key, 0).to_i
+      end
+    end
+
+    def normalized_expected_record_counts
+      (expected_record_counts || {}).to_h.transform_keys(&:to_s).transform_values(&:to_i)
+    end
+
+    def normalized_readback_verification
+      (readback_verification || {}).to_h.deep_stringify_keys
+    end
+
     def ndjson_blob_string
       blob_id = ndjson_file.blob&.id
 

app/views/api/v1/imports/show.json.jbuilder

@@ -44,6 +44,8 @@ json.data do
     json.unassigned_mappings_count mapping_counts[:unassigned_mappings_count]
   end
 
+  json.verification @import.verification_payload if @import.is_a?(SureImport)
+
   # Only show a subset of rows for preview if needed, or link to a separate rows endpoint
   # json.sample_rows @import.rows.limit(5)
 end

app/views/imports/_success.html.erb

@@ -11,6 +11,39 @@
       <p class="text-sm text-secondary"><%= t(".description") %></p>
     </div>
 
+    <% if import.is_a?(SureImport) %>
+      <% verification = import_verification_view(import) %>
+
+      <div class="bg-container border border-primary rounded-lg p-4 space-y-3 text-left">
+        <div class="flex items-center justify-between gap-3">
+          <h2 class="text-sm font-medium text-primary"><%= t(".verification.title") %></h2>
+          <span class="text-xs font-medium text-secondary"><%= t(".verification.status.#{verification.status}", default: verification.status.humanize) %></span>
+        </div>
+
+        <div class="grid grid-cols-2 gap-3 text-sm">
+          <div>
+            <p class="text-xs text-secondary"><%= t(".verification.checked") %></p>
+            <p class="font-medium text-primary"><%= verification.checked_total %></p>
+          </div>
+          <div>
+            <p class="text-xs text-secondary"><%= t(".verification.mismatches") %></p>
+            <p class="font-medium text-primary"><%= verification.mismatches_count %></p>
+          </div>
+        </div>
+
+        <% if verification.mismatches? %>
+          <div class="space-y-2">
+            <% verification.mismatches_preview.each do |key, counts| %>
+              <div class="flex items-center justify-between gap-3 text-xs">
+                <span class="text-secondary"><%= key.humanize %></span>
+                <span class="font-medium text-primary"><%= counts["actual"] %> / <%= counts["expected"] %></span>
+              </div>
+            <% end %>
+          </div>
+        <% end %>
+      </div>
+    <% end %>
+
     <%= render DS::Link.new(
       text: t(".back_to_dashboard"),
       variant: "primary",

config/locales/views/imports/en.yml

@@ -217,6 +217,22 @@ en:
       account: "Account"
       category: "Category"
       tag: "Tag"
+    dry_run_resources:
+      transactions: "Transactions"
+      balances: "Balances"
+      accounts: "Accounts"
+      categories: "Categories"
+      tags: "Tags"
+      rules: "Rules"
+      merchants: "Merchants"
+      recurring_transactions: "Recurring Transactions"
+      transfers: "Transfers"
+      rejected_transfers: "Rejected Transfers"
+      trades: "Trades"
+      holdings: "Holdings"
+      valuations: "Valuations"
+      budgets: "Budgets"
+      budget_categories: "Budget Categories"
     column_labels:
       date: "Date"
       amount: "Amount"
@@ -255,6 +271,16 @@ en:
       title: Import successful
       description: Your imported data has been successfully added to the app and is now ready for use.
       back_to_dashboard: Back to dashboard
+      verification:
+        title: Readback verification
+        checked: Checked
+        mismatches: Mismatches
+        status:
+          not_verified: Not verified
+          matched: Matched
+          mismatch: Mismatch
+          failed: Failed
+          reverted: Reverted
     importing:
       title: Import in progress
       description: "Your import is in progress. Check the imports menu for status updates or click 'Check Status' to refresh the page for updates. Feel free to continue using the app."

db/migrate/20260519100000_add_sure_import_verification_to_imports.rb

@@ -0,0 +1,6 @@
+class AddSureImportVerificationToImports < ActiveRecord::Migration[7.2]
+  def change
+    add_column :imports, :expected_record_counts, :jsonb, null: false, default: {}
+    add_column :imports, :readback_verification, :jsonb, null: false, default: {}
+  end
+end