diff --git a/app/controllers/api/v1/base_controller.rb b/app/controllers/api/v1/base_controller.rb index 4db722b52..ef175bf17 100644 --- a/app/controllers/api/v1/base_controller.rb +++ b/app/controllers/api/v1/base_controller.rb @@ -8,6 +8,12 @@ class Api::V1::BaseController < ApplicationController InvalidFilterError = Class.new(StandardError) + class << self + def valid_uuid?(value) + value.to_s.match?(UUID_PATTERN) + end + end + # Skip regular session-based authentication for API skip_authentication @@ -220,7 +226,7 @@ def render_json(data, status: :ok) end def valid_uuid?(value) - value.to_s.match?(UUID_PATTERN) + self.class.valid_uuid?(value) end def safe_page_param diff --git a/app/controllers/api/v1/imports_controller.rb b/app/controllers/api/v1/imports_controller.rb index 23b93ae7d..5f1e9cf3e 100644 --- a/app/controllers/api/v1/imports_controller.rb +++ b/app/controllers/api/v1/imports_controller.rb @@ -4,7 +4,7 @@ class Api::V1::ImportsController < Api::V1::BaseController include Pagy::Backend # Ensure proper scope authorization - before_action :ensure_read_scope, only: [ :index, :show, :rows ] + before_action :ensure_read_scope, only: [ :index, :show, :rows, :preflight ] before_action :ensure_write_scope, only: [ :create ] before_action :set_import_with_rows, only: [ :show ] before_action :set_import, only: [ :rows ] @@ -77,10 +77,10 @@ def create if params[:file].present? file = params[:file] - if file.size > Import::MAX_CSV_SIZE + if file.size > Import.max_csv_size return render json: { error: "file_too_large", - message: "File is too large. Maximum size is #{Import::MAX_CSV_SIZE / 1.megabyte}MB." + message: "File is too large. Maximum size is #{Import.max_csv_size / 1.megabyte}MB." }, status: :unprocessable_entity end @@ -93,10 +93,10 @@ def create @import.raw_file_str = file.read elsif params[:raw_file_content].present? - if params[:raw_file_content].bytesize > Import::MAX_CSV_SIZE + if params[:raw_file_content].bytesize > Import.max_csv_size return render json: { error: "content_too_large", - message: "Content is too large. Maximum size is #{Import::MAX_CSV_SIZE / 1.megabyte}MB." + message: "Content is too large. Maximum size is #{Import.max_csv_size / 1.megabyte}MB." }, status: :unprocessable_entity end @@ -136,6 +136,30 @@ def create render json: { error: "internal_server_error", message: e.message }, status: :internal_server_error end + def preflight + preflight_result = Import::Preflight.new(family: current_resource_owner.family, params: preflight_params).call + render json: preflight_result.payload, status: preflight_result.status + rescue ActiveRecord::RecordNotFound + render json: { + error: "record_not_found", + message: "The requested resource was not found" + }, status: :not_found + rescue CSV::MalformedCSVError => e + render json: { + error: "invalid_csv", + message: "CSV content could not be parsed", + errors: [ e.message ] + }, status: :unprocessable_entity + rescue StandardError => e + Rails.logger.error "ImportsController#preflight error: #{e.message}" + e.backtrace&.each { |line| Rails.logger.error line } + + render json: { + error: "internal_server_error", + message: "Error: #{e.message}" + }, status: :internal_server_error + end + private def set_import @@ -186,10 +210,15 @@ def import_config_params :signage_convention, :col_sep, :amount_type_strategy, - :amount_type_inflow_value + :amount_type_inflow_value, + :rows_to_skip ) end + def preflight_params + params.permit(*Import::Preflight::PARAM_KEYS) + end + def create_sure_import(family) content, filename, content_type = sure_import_upload_attributes return unless content @@ -282,10 +311,10 @@ def sure_import_upload_attributes end def sure_import_file_upload_attributes(file) - if file.size > SureImport::MAX_NDJSON_SIZE + if file.size > SureImport.max_ndjson_size render json: { error: "file_too_large", - message: "File is too large. Maximum size is #{SureImport::MAX_NDJSON_SIZE / 1.megabyte}MB." + message: "File is too large. Maximum size is #{SureImport.max_ndjson_size / 1.megabyte}MB." }, status: :unprocessable_entity return end @@ -308,10 +337,10 @@ def sure_import_file_upload_attributes(file) end def sure_import_raw_content_attributes(content) - if content.bytesize > SureImport::MAX_NDJSON_SIZE + if content.bytesize > SureImport.max_ndjson_size render json: { error: "content_too_large", - message: "Content is too large. Maximum size is #{SureImport::MAX_NDJSON_SIZE / 1.megabyte}MB." + message: "Content is too large. Maximum size is #{SureImport.max_ndjson_size / 1.megabyte}MB." }, status: :unprocessable_entity return end diff --git a/app/models/import.rb b/app/models/import.rb index fba9c8c32..24b0aab71 100644 --- a/app/models/import.rb +++ b/app/models/import.rb @@ -2,6 +2,7 @@ class Import < ApplicationRecord MaxRowCountExceededError = Class.new(StandardError) MappingError = Class.new(StandardError) + # Shared CSV upload/content limit for web and API imports, including preflight. MAX_CSV_SIZE = 10.megabytes MAX_PDF_SIZE = 25.megabytes ALLOWED_CSV_MIME_TYPES = %w[text/csv text/plain application/vnd.ms-excel application/csv].freeze @@ -24,6 +25,10 @@ def self.reasonable_date_range Date.new(1970, 1, 1)..Date.today.next_year(5) end + def self.max_csv_size + MAX_CSV_SIZE + end + AMOUNT_TYPE_STRATEGIES = %w[signed_amount custom_column].freeze belongs_to :family diff --git a/app/models/import/preflight.rb b/app/models/import/preflight.rb new file mode 100644 index 000000000..ef9429246 --- /dev/null +++ b/app/models/import/preflight.rb @@ -0,0 +1,454 @@ +# frozen_string_literal: true + +class Import::Preflight + Response = Struct.new(:status, :payload, keyword_init: true) + + class PreflightError < StandardError + attr_reader :status, :payload + + def initialize(response) + @status = response.status + @payload = response.payload + super(response.payload[:message]) + end + end + + CONFIG_PARAM_KEYS = %i[ + date_col_label + amount_col_label + name_col_label + category_col_label + tags_col_label + notes_col_label + account_col_label + qty_col_label + ticker_col_label + price_col_label + entity_type_col_label + currency_col_label + exchange_operating_mic_col_label + date_format + number_format + signage_convention + col_sep + amount_type_strategy + amount_type_inflow_value + rows_to_skip + ].freeze + + PARAM_KEYS = ([ + :type, + :account_id, + :file, + :raw_file_content + ] + CONFIG_PARAM_KEYS).freeze + + UNSUPPORTED_PREFLIGHT_IMPORT_TYPES = %w[PdfImport QifImport].freeze + IMPORT_TYPES = (Import::TYPES - UNSUPPORTED_PREFLIGHT_IMPORT_TYPES).freeze + + def initialize(family:, params:) + @family = family + @params = params.to_h.symbolize_keys + end + + def call + type = preflight_import_type + return invalid_import_type_response unless type + + type == "SureImport" ? sure_import_response : csv_import_response(type) + rescue PreflightError => e + Response.new(status: e.status, payload: e.payload) + end + + private + attr_reader :family, :params + + def preflight_import_type + type = params[:type].to_s + return "TransactionImport" if type.blank? + + type if IMPORT_TYPES.include?(type) + end + + def invalid_import_type_response + Response.new( + status: :unprocessable_entity, + payload: { + error: "invalid_import_type", + message: "type must be one of: #{IMPORT_TYPES.join(', ')}" + } + ) + end + + def sure_import_response + upload_attributes = sure_import_upload_attributes + return missing_sure_content_response unless upload_attributes + + content, filename, content_type = upload_attributes + Response.new( + status: :ok, + payload: { + data: sure_import_preflight_payload(content, filename, content_type) + } + ) + end + + def csv_import_response(type) + upload_attributes = csv_upload_attributes + return missing_csv_content_response unless upload_attributes + + content, filename, content_type = upload_attributes + import = family.imports.build(import_config_params.merge(type: type, raw_file_str: content)) + import.account = preflight_account if params[:account_id].present? + apply_import_defaults(import) + + return unsupported_import_type_response unless import.requires_csv_workflow? + + unless import.valid? + return Response.new( + status: :ok, + payload: { + data: csv_preflight_payload( + import: import, + type: type, + filename: filename, + content_type: content_type, + content: content, + parsed_rows_count: 0, + csv_headers: [], + missing_required_headers: [], + errors: validation_errors(import), + warnings: [] + ) + } + ) + end + + csv_content = csv_content_for(import, content) + csv = Import.parse_csv_str(csv_content, col_sep: import.col_sep) + parsed_rows_count = csv.length + csv_headers = Array(csv.headers).compact + missing_required_headers = missing_required_headers(import, csv_headers) + errors = validation_errors(import) + + if missing_required_headers.any? + errors << { + code: "missing_required_headers", + message: "Missing required columns: #{missing_required_headers.join(', ')}" + } + end + + if parsed_rows_count.zero? + errors << { + code: "no_data_rows", + message: "No data rows were found." + } + end + + warnings = [] + warnings << "Row count exceeds this import type's publish limit." if parsed_rows_count > import.max_row_count + + Response.new( + status: :ok, + payload: { + data: csv_preflight_payload( + import: import, + type: type, + filename: filename, + content_type: content_type, + content: content, + parsed_rows_count: parsed_rows_count, + csv_headers: csv_headers, + missing_required_headers: missing_required_headers, + errors: errors, + warnings: warnings + ) + } + ) + end + + def import_config_params + params.slice(*CONFIG_PARAM_KEYS) + end + + def preflight_account + raise ActiveRecord::RecordNotFound unless Api::V1::BaseController.valid_uuid?(params[:account_id]) + + family.accounts.find(params[:account_id]) + end + + def csv_upload_attributes + if params[:file].present? + csv_file_upload_attributes(params[:file]) + elsif params[:raw_file_content].present? + csv_raw_content_attributes(params[:raw_file_content].to_s) + end + end + + def csv_file_upload_attributes(file) + raise_response csv_file_too_large_response if file.size > Import.max_csv_size + raise_response invalid_csv_file_type_response unless Import::ALLOWED_CSV_MIME_TYPES.include?(file.content_type) + + [ + file.read, + file.original_filename.presence || "import.csv", + file.content_type.presence || "text/csv" + ] + end + + def csv_raw_content_attributes(content) + raise_response csv_content_too_large_response if content.bytesize > Import.max_csv_size + + [ content, "import.csv", "text/csv" ] + end + + def sure_import_upload_attributes + if params[:file].present? + sure_import_file_upload_attributes(params[:file]) + elsif params[:raw_file_content].present? + sure_import_raw_content_attributes(params[:raw_file_content].to_s) + end + end + + def sure_import_file_upload_attributes(file) + raise_response sure_file_too_large_response if file.size > SureImport.max_ndjson_size + + extension = File.extname(file.original_filename.to_s).downcase + unless SureImport::ALLOWED_NDJSON_CONTENT_TYPES.include?(file.content_type) || extension.in?(%w[.ndjson .json]) + raise_response invalid_sure_file_type_response + end + + [ + file.read, + file.original_filename.presence || "sure-import.ndjson", + file.content_type.presence || "application/x-ndjson" + ] + end + + def sure_import_raw_content_attributes(content) + raise_response sure_content_too_large_response if content.bytesize > SureImport.max_ndjson_size + + [ content, "sure-import.ndjson", "application/x-ndjson" ] + end + + def sure_import_preflight_payload(content, filename, content_type) + line_counts = Hash.new(0) + errors = [] + valid_rows_count = 0 + nonblank_rows_count = 0 + + content.each_line.with_index(1) do |line, line_number| + next if line.strip.blank? + + nonblank_rows_count += 1 + record = JSON.parse(line) + + unless record.is_a?(Hash) + errors << { + code: "invalid_ndjson_record", + message: "Line #{line_number} must be a JSON object." + } + next + end + + if record["type"].blank? || !record.key?("data") + errors << { + code: "invalid_ndjson_record", + message: "Line #{line_number} must include type and data." + } + next + end + + valid_rows_count += 1 + line_counts[record["type"]] += 1 + rescue JSON::ParserError => e + errors << { + code: "invalid_json", + message: "Line #{line_number} is not valid JSON: #{e.message}" + } + end + + if nonblank_rows_count.zero? + errors << { + code: "no_data_rows", + message: "No data rows were found." + } + end + + entity_counts = SureImport.dry_run_totals_from_line_type_counts(line_counts) + unsupported_types = line_counts.keys - SureImport.importable_ndjson_types + warnings = [] + warnings << "No importable records were found." if nonblank_rows_count.positive? && entity_counts.values.sum.zero? + warnings << "Some records use unsupported types: #{unsupported_types.join(', ')}" if unsupported_types.any? + warnings << "Row count exceeds this import type's publish limit." if nonblank_rows_count > SureImport.max_row_count + + { + type: "SureImport", + valid: errors.empty?, + content: content_payload(filename, content_type, content), + stats: { + rows_count: nonblank_rows_count, + valid_rows_count: valid_rows_count, + invalid_rows_count: nonblank_rows_count - valid_rows_count, + entity_counts: entity_counts, + record_type_counts: line_counts + }, + errors: errors, + warnings: warnings + } + end + + def content_payload(filename, content_type, content) + { + filename: filename, + content_type: content_type, + byte_size: content.bytesize + } + end + + def csv_content_for(import, content) + return content unless import.rows_to_skip.to_i.positive? + + content.lines.drop(import.rows_to_skip.to_i).join + end + + def apply_import_defaults(import) + return unless import.is_a?(MintImport) + + MintImport.default_column_mappings.each do |attribute, value| + import.public_send("#{attribute}=", value) if import.public_send(attribute).blank? + end + end + + def validation_errors(import) + import.errors.full_messages.map { |message| { code: "validation_failed", message: message } } + end + + def csv_preflight_payload(import:, type:, filename:, content_type:, content:, parsed_rows_count:, csv_headers:, missing_required_headers:, errors:, warnings:) + { + type: type, + valid: errors.empty?, + content: content_payload(filename, content_type, content), + stats: { + rows_count: parsed_rows_count + }, + headers: csv_headers, + required_headers: required_header_labels(import), + missing_required_headers: missing_required_headers, + errors: errors, + warnings: warnings + } + end + + def required_header_labels(import) + import.required_column_keys.filter_map do |key| + import.respond_to?("#{key}_col_label") ? import.public_send("#{key}_col_label").presence || key.to_s : key.to_s + end + end + + def missing_required_headers(import, headers) + normalized_headers = Array(headers).compact.to_h { |header| [ normalized_header(header), header ] } + + required_header_labels(import).reject do |header| + normalized_headers.key?(normalized_header(header)) + end + end + + def normalized_header(header) + header.to_s.strip.downcase.gsub(/\*/, "").gsub(/[\s-]+/, "_") + end + + def missing_csv_content_response + Response.new( + status: :unprocessable_entity, + payload: { + error: "missing_content", + message: "Provide a CSV file or raw_file_content." + } + ) + end + + def missing_sure_content_response + Response.new( + status: :unprocessable_entity, + payload: { + error: "missing_content", + message: "Provide a Sure NDJSON file or raw_file_content." + } + ) + end + + def csv_file_too_large_response + Response.new( + status: :unprocessable_entity, + payload: { + error: "file_too_large", + message: "File is too large. Maximum size is #{Import.max_csv_size / 1.megabyte}MB." + } + ) + end + + def csv_content_too_large_response + Response.new( + status: :unprocessable_entity, + payload: { + error: "content_too_large", + message: "Content is too large. Maximum size is #{Import.max_csv_size / 1.megabyte}MB." + } + ) + end + + def invalid_csv_file_type_response + Response.new( + status: :unprocessable_entity, + payload: { + error: "invalid_file_type", + message: "Invalid file type. Please upload a CSV file." + } + ) + end + + def sure_file_too_large_response + Response.new( + status: :unprocessable_entity, + payload: { + error: "file_too_large", + message: "File is too large. Maximum size is #{SureImport.max_ndjson_size / 1.megabyte}MB." + } + ) + end + + def sure_content_too_large_response + Response.new( + status: :unprocessable_entity, + payload: { + error: "content_too_large", + message: "Content is too large. Maximum size is #{SureImport.max_ndjson_size / 1.megabyte}MB." + } + ) + end + + def invalid_sure_file_type_response + Response.new( + status: :unprocessable_entity, + payload: { + error: "invalid_file_type", + message: "Invalid file type. Please upload a Sure NDJSON file." + } + ) + end + + def raise_response(response) + raise PreflightError, response + end + + def unsupported_import_type_response + Response.new( + status: :unprocessable_entity, + payload: { + error: "unsupported_import_type", + message: "Preflight supports CSV import types and SureImport." + } + ) + end +end diff --git a/app/models/mint_import.rb b/app/models/mint_import.rb index 87997d400..1dc5e5fb5 100644 --- a/app/models/mint_import.rb +++ b/app/models/mint_import.rb @@ -1,6 +1,24 @@ class MintImport < Import after_create :set_mappings + DEFAULT_COLUMN_MAPPINGS = { + signage_convention: "inflows_positive", + date_col_label: "Date", + date_format: "%m/%d/%Y", + name_col_label: "Description", + amount_col_label: "Amount", + currency_col_label: "Currency", + account_col_label: "Account Name", + category_col_label: "Category", + tags_col_label: "Labels", + notes_col_label: "Notes", + entity_type_col_label: "Transaction Type" + }.freeze + + def self.default_column_mappings + DEFAULT_COLUMN_MAPPINGS + end + def generate_rows_from_csv rows.destroy_all @@ -83,18 +101,7 @@ def signed_csv_amount(csv_row) private def set_mappings - self.signage_convention = "inflows_positive" - self.date_col_label = "Date" - self.date_format = "%m/%d/%Y" - self.name_col_label = "Description" - self.amount_col_label = "Amount" - self.currency_col_label = "Currency" - self.account_col_label = "Account Name" - self.category_col_label = "Category" - self.tags_col_label = "Labels" - self.notes_col_label = "Notes" - self.entity_type_col_label = "Transaction Type" - + assign_attributes(self.class.default_column_mappings) save! end end diff --git a/app/models/sure_import.rb b/app/models/sure_import.rb index 6666fc575..23815437a 100644 --- a/app/models/sure_import.rb +++ b/app/models/sure_import.rb @@ -1,5 +1,17 @@ class SureImport < Import MAX_NDJSON_SIZE = 10.megabytes + IMPORTABLE_NDJSON_TYPES = { + "Account" => :accounts, + "Category" => :categories, + "Tag" => :tags, + "Merchant" => :merchants, + "Transaction" => :transactions, + "Trade" => :trades, + "Valuation" => :valuations, + "Budget" => :budgets, + "BudgetCategory" => :budget_categories, + "Rule" => :rules + }.freeze ALLOWED_NDJSON_CONTENT_TYPES = %w[ application/x-ndjson application/ndjson @@ -11,6 +23,14 @@ class SureImport < Import has_one_attached :ndjson_file, dependent: :purge_later class << self + def max_row_count + 100_000 + end + + def max_ndjson_size + MAX_NDJSON_SIZE + end + # Counts JSON lines by top-level "type" (used for dry-run summaries and row limits). def ndjson_line_type_counts(content) return {} unless content.present? @@ -21,7 +41,7 @@ def ndjson_line_type_counts(content) begin record = JSON.parse(line) - counts[record["type"]] += 1 if record["type"] + counts[record["type"]] += 1 if record.is_a?(Hash) && record["type"] && record.key?("data") rescue JSON::ParserError # Skip invalid lines end @@ -30,19 +50,17 @@ def ndjson_line_type_counts(content) end def dry_run_totals_from_ndjson(content) - counts = ndjson_line_type_counts(content) - { - accounts: counts["Account"] || 0, - categories: counts["Category"] || 0, - tags: counts["Tag"] || 0, - merchants: counts["Merchant"] || 0, - transactions: counts["Transaction"] || 0, - trades: counts["Trade"] || 0, - valuations: counts["Valuation"] || 0, - budgets: counts["Budget"] || 0, - budget_categories: counts["BudgetCategory"] || 0, - rules: counts["Rule"] || 0 - } + dry_run_totals_from_line_type_counts(ndjson_line_type_counts(content)) + end + + def dry_run_totals_from_line_type_counts(counts) + IMPORTABLE_NDJSON_TYPES.to_h do |record_type, entity_key| + [ entity_key, counts[record_type] || 0 ] + end + end + + def importable_ndjson_types + IMPORTABLE_NDJSON_TYPES.keys end def valid_ndjson_first_line?(str) @@ -53,7 +71,7 @@ def valid_ndjson_first_line?(str) begin record = JSON.parse(first_line) - record.key?("type") && record.key?("data") + record.is_a?(Hash) && record.key?("type") && record.key?("data") rescue JSON::ParserError false end @@ -121,7 +139,7 @@ def publishable_from_validation_stats?(invalid_rows_count:) end def max_row_count - 100_000 + self.class.max_row_count end # Row total for max-row enforcement (counts every parsed line with a "type", including unsupported types). diff --git a/config/routes.rb b/config/routes.rb index 3a59f340b..93f8655e7 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -453,6 +453,7 @@ get :download, on: :member end resources :imports, only: [ :index, :show, :create ] do + post :preflight, on: :collection get :rows, on: :member end resource :usage, only: [ :show ], controller: :usage diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 9b2f867d9..6ac0a7d8a 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -1727,6 +1727,114 @@ components: unassigned_mappings_count: type: integer minimum: 0 + ImportPreflightContent: + type: object + required: + - filename + - content_type + - byte_size + properties: + filename: + type: string + content_type: + type: string + byte_size: + type: integer + minimum: 0 + ImportPreflightError: + type: object + required: + - code + - message + properties: + code: + type: string + message: + type: string + ImportPreflightStats: + type: object + required: + - rows_count + properties: + rows_count: + type: integer + minimum: 0 + description: CSV parsed non-header rows, or nonblank Sure NDJSON lines. + valid_rows_count: + type: integer + minimum: 0 + description: SureImport only. Valid NDJSON records. + invalid_rows_count: + type: integer + minimum: 0 + description: SureImport only. Invalid NDJSON records. CSV malformed content + returns a 422 instead. + entity_counts: + type: object + additionalProperties: + type: integer + nullable: true + record_type_counts: + type: object + additionalProperties: + type: integer + nullable: true + ImportPreflight: + type: object + required: + - type + - valid + - content + - stats + - errors + - warnings + properties: + type: + type: string + enum: + - TransactionImport + - TradeImport + - AccountImport + - MintImport + - CategoryImport + - RuleImport + - SureImport + valid: + type: boolean + content: + "$ref": "#/components/schemas/ImportPreflightContent" + stats: + "$ref": "#/components/schemas/ImportPreflightStats" + headers: + type: array + items: + type: string + nullable: true + required_headers: + type: array + items: + type: string + nullable: true + missing_required_headers: + type: array + items: + type: string + nullable: true + errors: + type: array + items: + "$ref": "#/components/schemas/ImportPreflightError" + warnings: + type: array + items: + type: string + ImportPreflightResponse: + type: object + required: + - data + properties: + data: + "$ref": "#/components/schemas/ImportPreflight" ImportStatusSummary: type: object required: @@ -4381,7 +4489,7 @@ paths: post: summary: Create import description: Create a new import from raw CSV content, inline Sure NDJSON content, - or an uploaded Sure NDJSON file. + or an uploaded Sure NDJSON file. CSV content is limited to 10MB. tags: - Imports security: @@ -4416,8 +4524,9 @@ paths: properties: raw_file_content: type: string - description: Raw CSV or Sure NDJSON content as a string. Required - for SureImport unless a multipart file is uploaded. + description: Raw CSV or Sure NDJSON content as a string. CSV content + is limited to 10MB. Required for SureImport unless a multipart + file is uploaded. type: type: string enum: @@ -4521,8 +4630,9 @@ paths: properties: raw_file_content: type: string - description: Raw CSV or Sure NDJSON content as a string. Required - for SureImport unless a multipart file is uploaded. + description: Raw CSV or Sure NDJSON content as a string. CSV content + is limited to 10MB. Required for SureImport unless a multipart + file is uploaded. type: type: string enum: @@ -4709,6 +4819,264 @@ paths: application/json: schema: "$ref": "#/components/schemas/ErrorResponse" + "/api/v1/imports/preflight": + post: + summary: Validate import content without creating an import + description: Validate CSV or Sure NDJSON import content and return counts, headers, + warnings, and validation errors without persisting an import or enqueueing + jobs. CSV content is limited to 10MB. + tags: + - Imports + security: + - apiKeyAuth: [] + parameters: [] + responses: + '200': + description: import content preflighted + content: + application/json: + schema: + "$ref": "#/components/schemas/ImportPreflightResponse" + '401': + description: unauthorized + content: + application/json: + schema: + "$ref": "#/components/schemas/ErrorResponse" + '422': + description: missing or invalid content + content: + application/json: + schema: + "$ref": "#/components/schemas/ErrorResponse" + '404': + description: account not found + content: + application/json: + schema: + "$ref": "#/components/schemas/ErrorResponse" + requestBody: + content: + application/json: + schema: + type: object + properties: + raw_file_content: + type: string + description: Raw CSV or Sure NDJSON content as a string. CSV content + is limited to 10MB. + file: + type: string + format: binary + description: CSV or Sure NDJSON upload when using multipart/form-data. + CSV files are limited to 10MB. + type: + type: string + enum: + - TransactionImport + - TradeImport + - AccountImport + - MintImport + - CategoryImport + - RuleImport + - SureImport + description: Import type to validate (defaults to TransactionImport) + account_id: + type: string + format: uuid + description: Account ID used for account-scoped CSV import validation + date_col_label: + type: string + description: CSV imports only. Header name for the date column + amount_col_label: + type: string + description: CSV imports only. Header name for the amount column + name_col_label: + type: string + description: CSV imports only. Header name for the transaction name + column + category_col_label: + type: string + description: CSV imports only. Header name for the category column + tags_col_label: + type: string + description: CSV imports only. Header name for the tags column + notes_col_label: + type: string + description: CSV imports only. Header name for the notes column + account_col_label: + type: string + description: CSV imports only. Header name for the account column + qty_col_label: + type: string + description: CSV trade imports only. Header name for the quantity + column + ticker_col_label: + type: string + description: CSV trade imports only. Header name for the ticker + column + price_col_label: + type: string + description: CSV trade imports only. Header name for the price column + entity_type_col_label: + type: string + description: CSV imports only. Header name for the entity type column + currency_col_label: + type: string + description: CSV imports only. Header name for the currency column + exchange_operating_mic_col_label: + type: string + description: CSV trade imports only. Header name for the exchange + operating MIC column + date_format: + type: string + description: CSV imports only. Date format pattern + number_format: + type: string + enum: + - '1,234.56' + - 1.234,56 + - 1 234,56 + - '1,234' + description: CSV imports only. Number format for parsing amounts + signage_convention: + type: string + enum: + - inflows_positive + - inflows_negative + description: CSV imports only. How to interpret positive/negative + amounts + col_sep: + type: string + enum: + - "," + - ";" + description: CSV imports only. Column separator + rows_to_skip: + type: integer + minimum: 0 + description: CSV imports only. Number of leading rows to skip before + reading headers + amount_type_strategy: + type: string + enum: + - signed_amount + - custom_column + description: CSV imports only. Amount parsing strategy + amount_type_inflow_value: + type: string + description: CSV imports only. Column value that marks an amount + as an inflow when using custom_column strategy + multipart/form-data: + schema: + type: object + properties: + raw_file_content: + type: string + description: Raw CSV or Sure NDJSON content as a string. CSV content + is limited to 10MB. + file: + type: string + format: binary + description: CSV or Sure NDJSON upload when using multipart/form-data. + CSV files are limited to 10MB. + type: + type: string + enum: + - TransactionImport + - TradeImport + - AccountImport + - MintImport + - CategoryImport + - RuleImport + - SureImport + description: Import type to validate (defaults to TransactionImport) + account_id: + type: string + format: uuid + description: Account ID used for account-scoped CSV import validation + date_col_label: + type: string + description: CSV imports only. Header name for the date column + amount_col_label: + type: string + description: CSV imports only. Header name for the amount column + name_col_label: + type: string + description: CSV imports only. Header name for the transaction name + column + category_col_label: + type: string + description: CSV imports only. Header name for the category column + tags_col_label: + type: string + description: CSV imports only. Header name for the tags column + notes_col_label: + type: string + description: CSV imports only. Header name for the notes column + account_col_label: + type: string + description: CSV imports only. Header name for the account column + qty_col_label: + type: string + description: CSV trade imports only. Header name for the quantity + column + ticker_col_label: + type: string + description: CSV trade imports only. Header name for the ticker + column + price_col_label: + type: string + description: CSV trade imports only. Header name for the price column + entity_type_col_label: + type: string + description: CSV imports only. Header name for the entity type column + currency_col_label: + type: string + description: CSV imports only. Header name for the currency column + exchange_operating_mic_col_label: + type: string + description: CSV trade imports only. Header name for the exchange + operating MIC column + date_format: + type: string + description: CSV imports only. Date format pattern + number_format: + type: string + enum: + - '1,234.56' + - 1.234,56 + - 1 234,56 + - '1,234' + description: CSV imports only. Number format for parsing amounts + signage_convention: + type: string + enum: + - inflows_positive + - inflows_negative + description: CSV imports only. How to interpret positive/negative + amounts + col_sep: + type: string + enum: + - "," + - ";" + description: CSV imports only. Column separator + rows_to_skip: + type: integer + minimum: 0 + description: CSV imports only. Number of leading rows to skip before + reading headers + amount_type_strategy: + type: string + enum: + - signed_amount + - custom_column + description: CSV imports only. Amount parsing strategy + amount_type_inflow_value: + type: string + description: CSV imports only. Column value that marks an amount + as an inflow when using custom_column strategy "/api/v1/merchants": get: summary: List merchants diff --git a/spec/requests/api/v1/imports_spec.rb b/spec/requests/api/v1/imports_spec.rb index 540422c33..a12cb7a8c 100644 --- a/spec/requests/api/v1/imports_spec.rb +++ b/spec/requests/api/v1/imports_spec.rb @@ -123,7 +123,7 @@ end post 'Create import' do - description 'Create a new import from raw CSV content, inline Sure NDJSON content, or an uploaded Sure NDJSON file.' + description 'Create a new import from raw CSV content, inline Sure NDJSON content, or an uploaded Sure NDJSON file. CSV content is limited to 10MB.' tags 'Imports' security [ { apiKeyAuth: [] } ] consumes 'application/json', 'multipart/form-data' @@ -134,7 +134,7 @@ properties: { raw_file_content: { type: :string, - description: 'Raw CSV or Sure NDJSON content as a string. Required for SureImport unless a multipart file is uploaded.' + description: 'Raw CSV or Sure NDJSON content as a string. CSV content is limited to 10MB. Required for SureImport unless a multipart file is uploaded.' }, type: { type: :string, @@ -365,4 +365,126 @@ end end end + + path '/api/v1/imports/preflight' do + post 'Validate import content without creating an import' do + description 'Validate CSV or Sure NDJSON import content and return counts, headers, warnings, and validation errors without persisting an import or enqueueing jobs. CSV content is limited to 10MB.' + tags 'Imports' + security [ { apiKeyAuth: [] } ] + consumes 'application/json', 'multipart/form-data' + produces 'application/json' + + parameter name: :body, in: :body, required: false, schema: { + type: :object, + properties: { + raw_file_content: { + type: :string, + description: 'Raw CSV or Sure NDJSON content as a string. CSV content is limited to 10MB.' + }, + file: { + type: :string, + format: :binary, + description: 'CSV or Sure NDJSON upload when using multipart/form-data. CSV files are limited to 10MB.' + }, + type: { + type: :string, + enum: %w[TransactionImport TradeImport AccountImport MintImport CategoryImport RuleImport SureImport], + description: 'Import type to validate (defaults to TransactionImport)' + }, + account_id: { + type: :string, + format: :uuid, + description: 'Account ID used for account-scoped CSV import validation' + }, + date_col_label: { type: :string, description: 'CSV imports only. Header name for the date column' }, + amount_col_label: { type: :string, description: 'CSV imports only. Header name for the amount column' }, + name_col_label: { type: :string, description: 'CSV imports only. Header name for the transaction name column' }, + category_col_label: { type: :string, description: 'CSV imports only. Header name for the category column' }, + tags_col_label: { type: :string, description: 'CSV imports only. Header name for the tags column' }, + notes_col_label: { type: :string, description: 'CSV imports only. Header name for the notes column' }, + account_col_label: { type: :string, description: 'CSV imports only. Header name for the account column' }, + qty_col_label: { type: :string, description: 'CSV trade imports only. Header name for the quantity column' }, + ticker_col_label: { type: :string, description: 'CSV trade imports only. Header name for the ticker column' }, + price_col_label: { type: :string, description: 'CSV trade imports only. Header name for the price column' }, + entity_type_col_label: { type: :string, description: 'CSV imports only. Header name for the entity type column' }, + currency_col_label: { type: :string, description: 'CSV imports only. Header name for the currency column' }, + exchange_operating_mic_col_label: { type: :string, description: 'CSV trade imports only. Header name for the exchange operating MIC column' }, + date_format: { type: :string, description: 'CSV imports only. Date format pattern' }, + number_format: { + type: :string, + enum: [ '1,234.56', '1.234,56', '1 234,56', '1,234' ], + description: 'CSV imports only. Number format for parsing amounts' + }, + signage_convention: { + type: :string, + enum: %w[inflows_positive inflows_negative], + description: 'CSV imports only. How to interpret positive/negative amounts' + }, + col_sep: { + type: :string, + enum: [ ',', ';' ], + description: 'CSV imports only. Column separator' + }, + rows_to_skip: { + type: :integer, + minimum: 0, + description: 'CSV imports only. Number of leading rows to skip before reading headers' + }, + amount_type_strategy: { + type: :string, + enum: %w[signed_amount custom_column], + description: 'CSV imports only. Amount parsing strategy' + }, + amount_type_inflow_value: { + type: :string, + description: 'CSV imports only. Column value that marks an amount as an inflow when using custom_column strategy' + } + } + } + + response '200', 'import content preflighted' do + schema '$ref' => '#/components/schemas/ImportPreflightResponse' + + let(:body) do + { + raw_file_content: "date,amount,name\n01/15/2024,50.00,New Transaction", + type: 'TransactionImport', + account_id: account.id, + date_col_label: 'date', + amount_col_label: 'amount', + name_col_label: 'name' + } + end + + run_test! + end + + response '401', 'unauthorized' do + schema '$ref' => '#/components/schemas/ErrorResponse' + let(:'X-Api-Key') { nil } + let(:body) { { raw_file_content: "date,amount\n01/15/2024,50.00" } } + + run_test! + end + + response '422', 'missing or invalid content' do + schema '$ref' => '#/components/schemas/ErrorResponse' + let(:body) { { type: 'SureImport' } } + + run_test! + end + + response '404', 'account not found' do + schema '$ref' => '#/components/schemas/ErrorResponse' + let(:body) do + { + raw_file_content: "date,amount,name\n01/15/2024,50.00,New Transaction", + account_id: SecureRandom.uuid + } + end + + run_test! + end + end + end end diff --git a/spec/swagger_helper.rb b/spec/swagger_helper.rb index d8260c27e..febeb887b 100644 --- a/spec/swagger_helper.rb +++ b/spec/swagger_helper.rb @@ -945,6 +945,94 @@ unassigned_mappings_count: { type: :integer, minimum: 0 } } }, + ImportPreflightContent: { + type: :object, + required: %w[filename content_type byte_size], + properties: { + filename: { type: :string }, + content_type: { type: :string }, + byte_size: { type: :integer, minimum: 0 } + } + }, + ImportPreflightError: { + type: :object, + required: %w[code message], + properties: { + code: { type: :string }, + message: { type: :string } + } + }, + ImportPreflightStats: { + type: :object, + required: %w[rows_count], + properties: { + rows_count: { + type: :integer, + minimum: 0, + description: 'CSV parsed non-header rows, or nonblank Sure NDJSON lines.' + }, + valid_rows_count: { + type: :integer, + minimum: 0, + description: 'SureImport only. Valid NDJSON records.' + }, + invalid_rows_count: { + type: :integer, + minimum: 0, + description: 'SureImport only. Invalid NDJSON records. CSV malformed content returns a 422 instead.' + }, + entity_counts: { + type: :object, + additionalProperties: { type: :integer }, + nullable: true + }, + record_type_counts: { + type: :object, + additionalProperties: { type: :integer }, + nullable: true + } + } + }, + ImportPreflight: { + type: :object, + required: %w[type valid content stats errors warnings], + properties: { + type: { type: :string, enum: %w[TransactionImport TradeImport AccountImport MintImport CategoryImport RuleImport SureImport] }, + valid: { type: :boolean }, + content: { '$ref' => '#/components/schemas/ImportPreflightContent' }, + stats: { '$ref' => '#/components/schemas/ImportPreflightStats' }, + headers: { + type: :array, + items: { type: :string }, + nullable: true + }, + required_headers: { + type: :array, + items: { type: :string }, + nullable: true + }, + missing_required_headers: { + type: :array, + items: { type: :string }, + nullable: true + }, + errors: { + type: :array, + items: { '$ref' => '#/components/schemas/ImportPreflightError' } + }, + warnings: { + type: :array, + items: { type: :string } + } + } + }, + ImportPreflightResponse: { + type: :object, + required: %w[data], + properties: { + data: { '$ref' => '#/components/schemas/ImportPreflight' } + } + }, ImportStatusSummary: { type: :object, required: %w[uploaded configured terminal], diff --git a/test/controllers/api/v1/imports_controller_test.rb b/test/controllers/api/v1/imports_controller_test.rb index bab5e3787..d414e7e1b 100644 --- a/test/controllers/api/v1/imports_controller_test.rb +++ b/test/controllers/api/v1/imports_controller_test.rb @@ -405,9 +405,7 @@ class Api::V1::ImportsControllerTest < ActionDispatch::IntegrationTest original_filename: "large.ndjson" ) - original_value = SureImport::MAX_NDJSON_SIZE - SureImport.send(:remove_const, :MAX_NDJSON_SIZE) - SureImport.const_set(:MAX_NDJSON_SIZE, test_limit) + SureImport.stubs(:max_ndjson_size).returns(test_limit) assert_no_difference("Import.count") do post api_v1_imports_url, @@ -421,9 +419,6 @@ class Api::V1::ImportsControllerTest < ActionDispatch::IntegrationTest assert_response :unprocessable_entity json_response = JSON.parse(response.body) assert_equal "file_too_large", json_response["error"] - ensure - SureImport.send(:remove_const, :MAX_NDJSON_SIZE) - SureImport.const_set(:MAX_NDJSON_SIZE, original_value) end test "should reject Sure import uploaded file with invalid type" do @@ -551,6 +546,473 @@ class Api::V1::ImportsControllerTest < ActionDispatch::IntegrationTest assert_equal "invalid_ndjson", json_response["error"] end + test "should preflight CSV import without persisting records" do + csv_content = "date,amount,name\n2023-01-01,-10.00,Test Transaction" + + assert_no_difference([ "Import.count", "Import::Row.count" ]) do + post preflight_api_v1_imports_url, + params: { + raw_file_content: csv_content, + date_col_label: "date", + amount_col_label: "amount", + name_col_label: "name", + account_id: @account.id + }, + headers: api_headers(@api_key) + end + + assert_response :success + json_response = JSON.parse(response.body) + data = json_response["data"] + + assert_equal "TransactionImport", data["type"] + assert_equal true, data["valid"] + assert_equal 1, data["stats"]["rows_count"] + assert_not data["stats"].key?("valid_rows_count") + assert_not data["stats"].key?("invalid_rows_count") + assert_equal %w[date amount name], data["headers"] + assert_empty data["missing_required_headers"] + assert_empty data["errors"] + end + + test "should report missing required CSV headers during preflight" do + csv_content = "name\nMissing Amount" + + assert_no_difference("Import.count") do + post preflight_api_v1_imports_url, + params: { + raw_file_content: csv_content, + date_col_label: "date", + amount_col_label: "amount", + name_col_label: "name", + account_id: @account.id + }, + headers: api_headers(@api_key) + end + + assert_response :success + data = JSON.parse(response.body)["data"] + + assert_equal false, data["valid"] + assert_equal 1, data["stats"]["rows_count"] + assert_not data["stats"].key?("valid_rows_count") + assert_not data["stats"].key?("invalid_rows_count") + assert_equal [ "date", "amount" ], data["missing_required_headers"] + assert_equal "missing_required_headers", data["errors"].first["code"] + end + + test "should apply rows_to_skip before CSV preflight header validation" do + csv_content = [ + "Generated by bank export", + "posted,amount,description", + "2024-01-01,-10.00,Coffee" + ].join("\n") + + assert_no_difference("Import.count") do + post preflight_api_v1_imports_url, + params: { + raw_file_content: csv_content, + rows_to_skip: 1, + date_col_label: "posted", + amount_col_label: "amount", + name_col_label: "description", + account_id: @account.id + }, + headers: api_headers(@read_only_api_key) + end + + assert_response :success + data = JSON.parse(response.body)["data"] + + assert_equal true, data["valid"] + assert_equal 1, data["stats"]["rows_count"] + assert_equal %w[posted amount description], data["headers"] + assert_empty data["missing_required_headers"] + end + + test "should preflight semicolon separated CSV content" do + csv_content = "date;amount;name\n2024-01-01;-10.00;Coffee" + + assert_no_difference("Import.count") do + post preflight_api_v1_imports_url, + params: { + raw_file_content: csv_content, + col_sep: ";", + date_col_label: "date", + amount_col_label: "amount", + name_col_label: "name", + account_id: @account.id + }, + headers: api_headers(@read_only_api_key) + end + + assert_response :success + data = JSON.parse(response.body)["data"] + + assert_equal true, data["valid"] + assert_equal 1, data["stats"]["rows_count"] + assert_equal %w[date amount name], data["headers"] + end + + test "should report invalid preflight CSV parser config without parsing" do + csv_content = "date,amount,name\n2024-01-01,-10.00,Coffee" + + assert_no_difference("Import.count") do + post preflight_api_v1_imports_url, + params: { + raw_file_content: csv_content, + col_sep: "", + date_col_label: "date", + amount_col_label: "amount", + name_col_label: "name", + account_id: @account.id + }, + headers: api_headers(@read_only_api_key) + end + + assert_response :success + data = JSON.parse(response.body)["data"] + + assert_equal false, data["valid"] + assert_equal 0, data["stats"]["rows_count"] + assert_empty data["headers"] + assert_equal "validation_failed", data["errors"].first["code"] + end + + test "should reject malformed CSV during preflight" do + csv_content = "date,amount,name\n2024-01-01,-10.00,\"Coffee Shop" + + assert_no_difference("Import.count") do + post preflight_api_v1_imports_url, + params: { + raw_file_content: csv_content, + date_col_label: "date", + amount_col_label: "amount", + name_col_label: "name", + account_id: @account.id + }, + headers: api_headers(@read_only_api_key) + end + + assert_response :unprocessable_entity + json_response = JSON.parse(response.body) + assert_equal "invalid_csv", json_response["error"] + end + + test "should include preflight exception message in internal server error response" do + Import::Preflight.any_instance.stubs(:call).raises(StandardError, "boom") + + post preflight_api_v1_imports_url, + params: { + raw_file_content: "date,amount,name\n2024-01-01,-10.00,Coffee", + date_col_label: "date", + amount_col_label: "amount", + name_col_label: "name" + }, + headers: api_headers(@read_only_api_key) + + assert_response :internal_server_error + json_response = JSON.parse(response.body) + assert_equal "internal_server_error", json_response["error"] + assert_equal "Error: boom", json_response["message"] + end + + test "should reject unknown preflight import type" do + assert_no_difference("Import.count") do + post preflight_api_v1_imports_url, + params: { + type: "FakeImport", + raw_file_content: "date,amount,name\n2023-01-01,-10.00,Test Transaction" + }, + headers: api_headers(@read_only_api_key) + end + + assert_response :unprocessable_entity + response_data = JSON.parse(response.body) + assert_equal "invalid_import_type", response_data["error"] + assert_not response_data.key?("errors") + end + + test "should reject import types excluded from preflight" do + assert_no_difference("Import.count") do + post preflight_api_v1_imports_url, + params: { + type: "QifImport", + raw_file_content: "!Type:Bank\nD01/01/2024\nT-10.00\nPTest\n^" + }, + headers: api_headers(@read_only_api_key) + end + + assert_response :unprocessable_entity + response_data = JSON.parse(response.body) + assert_equal "invalid_import_type", response_data["error"] + assert_not response_data.key?("errors") + assert_not_includes response_data["message"], "QifImport" + assert_not_includes response_data["message"], "PdfImport" + end + + test "should report empty CSV preflight content as invalid" do + assert_no_difference("Import.count") do + post preflight_api_v1_imports_url, + params: { + raw_file_content: "date,amount,name\n", + date_col_label: "date", + amount_col_label: "amount", + name_col_label: "name", + account_id: @account.id + }, + headers: api_headers(@read_only_api_key) + end + + assert_response :success + data = JSON.parse(response.body)["data"] + + assert_equal false, data["valid"] + assert_equal 0, data["stats"]["rows_count"] + assert_equal "no_data_rows", data["errors"].first["code"] + assert_empty data["warnings"] + end + + test "should preflight Sure import without persisting records" do + ndjson_content = [ + { type: "Account", data: { id: "account_1", name: "Checking" } }.to_json, + { type: "Transaction", data: { id: "entry_1", account_id: "account_1" } }.to_json + ].join("\n") + + assert_no_difference("Import.count") do + post preflight_api_v1_imports_url, + params: { + type: "SureImport", + raw_file_content: ndjson_content + }, + headers: api_headers(@api_key) + end + + assert_response :success + data = JSON.parse(response.body)["data"] + + assert_equal "SureImport", data["type"] + assert_equal true, data["valid"] + assert_equal 2, data["stats"]["rows_count"] + assert_equal 1, data["stats"]["entity_counts"]["accounts"] + assert_equal 1, data["stats"]["entity_counts"]["transactions"] + assert_empty data["errors"] + end + + test "should report invalid Sure import NDJSON during preflight" do + assert_no_difference("Import.count") do + post preflight_api_v1_imports_url, + params: { + type: "SureImport", + raw_file_content: "not ndjson" + }, + headers: api_headers(@api_key) + end + + assert_response :success + data = JSON.parse(response.body)["data"] + + assert_equal false, data["valid"] + assert_equal 1, data["stats"]["invalid_rows_count"] + assert_equal "invalid_json", data["errors"].first["code"] + end + + test "should report non-object Sure import NDJSON records during preflight" do + assert_no_difference("Import.count") do + post preflight_api_v1_imports_url, + params: { + type: "SureImport", + raw_file_content: "[]" + }, + headers: api_headers(@read_only_api_key) + end + + assert_response :success + data = JSON.parse(response.body)["data"] + + assert_equal false, data["valid"] + assert_equal 1, data["stats"]["invalid_rows_count"] + assert_equal "invalid_ndjson_record", data["errors"].first["code"] + end + + test "should report empty Sure import file as invalid during preflight" do + empty_file = Rack::Test::UploadedFile.new( + StringIO.new(""), + "application/x-ndjson", + original_filename: "empty.ndjson" + ) + + assert_no_difference("Import.count") do + post preflight_api_v1_imports_url, + params: { + type: "SureImport", + file: empty_file + }, + headers: api_headers(@read_only_api_key) + end + + assert_response :success + data = JSON.parse(response.body)["data"] + + assert_equal false, data["valid"] + assert_equal 0, data["stats"]["rows_count"] + assert_equal "no_data_rows", data["errors"].first["code"] + assert_empty data["warnings"] + end + + test "should reject preflight with no file or raw content" do + assert_no_difference("Import.count") do + post preflight_api_v1_imports_url, + params: { type: "SureImport" }, + headers: api_headers(@api_key) + end + + assert_response :unprocessable_entity + assert_equal "missing_content", JSON.parse(response.body)["error"] + end + + test "should reject oversized file uploads during preflight" do + test_limit = 1.kilobyte + large_file = Rack::Test::UploadedFile.new( + StringIO.new("x" * (test_limit + 1)), + "text/csv", + original_filename: "large.csv" + ) + + Import.stubs(:max_csv_size).returns(test_limit) + + assert_no_difference("Import.count") do + post preflight_api_v1_imports_url, + params: { file: large_file }, + headers: api_headers(@read_only_api_key) + end + + assert_response :unprocessable_entity + assert_equal "file_too_large", JSON.parse(response.body)["error"] + end + + test "should preflight with read-only API key" do + csv_content = "date,amount,name\n2023-01-01,-10.00,Test Transaction" + + assert_no_difference("Import.count") do + post preflight_api_v1_imports_url, + params: { + raw_file_content: csv_content, + date_col_label: "date", + amount_col_label: "amount", + name_col_label: "name", + account_id: @account.id + }, + headers: api_headers(@read_only_api_key) + end + + assert_response :success + assert_equal true, JSON.parse(response.body)["data"]["valid"] + end + + test "should require authentication for preflight" do + post preflight_api_v1_imports_url, params: { + raw_file_content: "date,amount,name\n2023-01-01,-10.00,Test Transaction" + } + + assert_response :unauthorized + end + + test "should return not found for preflight account outside family" do + other_family = Family.create!(name: "Other Family", currency: "USD", locale: "en") + other_depository = Depository.create!(subtype: "checking") + other_account = Account.create!( + family: other_family, + name: "Other Account", + currency: "USD", + classification: "asset", + accountable: other_depository, + balance: 0 + ) + + assert_no_difference("Import.count") do + post preflight_api_v1_imports_url, + params: { + raw_file_content: "date,amount,name\n2023-01-01,-10.00,Test Transaction", + date_col_label: "date", + amount_col_label: "amount", + name_col_label: "name", + account_id: other_account.id + }, + headers: api_headers(@read_only_api_key) + end + + assert_response :not_found + assert_equal "record_not_found", JSON.parse(response.body)["error"] + end + + test "should return not found for malformed preflight account id" do + assert_no_difference("Import.count") do + post preflight_api_v1_imports_url, + params: { + raw_file_content: "date,amount,name\n2023-01-01,-10.00,Test Transaction", + date_col_label: "date", + amount_col_label: "amount", + name_col_label: "name", + account_id: "not-a-uuid" + }, + headers: api_headers(@read_only_api_key) + end + + assert_response :not_found + assert_equal "record_not_found", JSON.parse(response.body)["error"] + end + + test "should apply Mint defaults before preflight header validation" do + mint_content = [ + "Date,Amount,Account Name,Description,Category,Labels,Currency,Notes,Transaction Type", + "01/01/2024,-8.55,Checking,Starbucks,Food & Drink,Coffee,USD,Morning coffee,debit" + ].join("\n") + + assert_no_difference("Import.count") do + post preflight_api_v1_imports_url, + params: { + type: "MintImport", + raw_file_content: mint_content + }, + headers: api_headers(@read_only_api_key) + end + + assert_response :success + data = JSON.parse(response.body)["data"] + + assert_equal "MintImport", data["type"] + assert_equal true, data["valid"] + assert_empty data["missing_required_headers"] + assert_includes data["required_headers"], "Date" + assert_includes data["required_headers"], "Amount" + end + + test "should not overwrite explicit Mint preflight column mappings with defaults" do + mint_content = [ + "Posted On,Value,Description", + "01/01/2024,-8.55,Starbucks" + ].join("\n") + + assert_no_difference("Import.count") do + post preflight_api_v1_imports_url, + params: { + type: "MintImport", + raw_file_content: mint_content, + date_col_label: "Posted On", + amount_col_label: "Value" + }, + headers: api_headers(@read_only_api_key) + end + + assert_response :success + data = JSON.parse(response.body)["data"] + + assert_equal true, data["valid"] + assert_equal [ "Posted On", "Value" ], data["required_headers"] + assert_empty data["missing_required_headers"] + end + test "should create import and auto-publish when configured and requested" do csv_content = "date,amount,name\n2023-01-01,-10.00,Test Transaction" @@ -633,9 +1095,7 @@ class Api::V1::ImportsControllerTest < ActionDispatch::IntegrationTest test_limit = 1.kilobyte large_content = "x" * (test_limit + 1) - original_value = Import::MAX_CSV_SIZE - Import.send(:remove_const, :MAX_CSV_SIZE) - Import.const_set(:MAX_CSV_SIZE, test_limit) + Import.stubs(:max_csv_size).returns(test_limit) assert_no_difference("Import.count") do post api_v1_imports_url, @@ -646,9 +1106,6 @@ class Api::V1::ImportsControllerTest < ActionDispatch::IntegrationTest assert_response :unprocessable_entity json_response = JSON.parse(response.body) assert_equal "content_too_large", json_response["error"] - ensure - Import.send(:remove_const, :MAX_CSV_SIZE) - Import.const_set(:MAX_CSV_SIZE, original_value) end test "should accept file upload with valid csv mime type" do diff --git a/test/models/mint_import_test.rb b/test/models/mint_import_test.rb index 15095cefa..599e18b10 100644 --- a/test/models/mint_import_test.rb +++ b/test/models/mint_import_test.rb @@ -5,6 +5,14 @@ class MintImportTest < ActiveSupport::TestCase @family = families(:dylan_family) end + test "default column mappings are applied after create" do + import = @family.imports.create!(type: "MintImport") + + MintImport.default_column_mappings.each do |attribute, value| + assert_equal value, import.public_send(attribute) + end + end + test "generated rows preserve stable source row numbers" do import = @family.imports.create!( type: "MintImport", diff --git a/test/models/sure_import_test.rb b/test/models/sure_import_test.rb index 65fbe483c..9345dea60 100644 --- a/test/models/sure_import_test.rb +++ b/test/models/sure_import_test.rb @@ -37,9 +37,37 @@ class SureImportTest < ActiveSupport::TestCase end test "max_row_count is higher than standard imports" do + assert_equal 100_000, SureImport.max_row_count assert_equal 100_000, @import.max_row_count end + test "dry_run totals can be derived from existing line type counts" do + counts = { + "Account" => 2, + "Transaction" => 3, + "UnknownType" => 4 + } + + dry_run = SureImport.dry_run_totals_from_line_type_counts(counts) + + assert_equal 2, dry_run[:accounts] + assert_equal 3, dry_run[:transactions] + assert_equal 0, dry_run[:categories] + assert_not dry_run.key?(:unknown_type) + end + + test "ndjson line type counts ignore records without data" do + ndjson = [ + { type: "Account", data: { id: "uuid-1" } }, + { type: "Transaction" }, + { data: { id: "uuid-2" } } + ].map(&:to_json).join("\n") + + counts = SureImport.ndjson_line_type_counts(ndjson) + + assert_equal({ "Account" => 1 }, counts) + end + test "csv_template returns nil" do assert_nil @import.csv_template end