Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 12 additions & 2 deletions app/parsers/concerns/bulkrax/csv_parser/csv_validation.rb
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ def validate_csv(csv_file:, zip_file: nil, admin_set_id: nil) # rubocop:disable
# 7. Header-level checks
missing_required = find_missing_required_headers(headers, field_metadata, mapping_manager)
unrecognized = find_unrecognized_validation_headers(headers, valid_headers)
empty_columns = find_empty_column_positions(headers, raw_csv)

# 8. Row-level validators
parent_split = resolve_parent_split_pattern(mappings)
Expand Down Expand Up @@ -82,12 +83,13 @@ def validate_csv(csv_file:, zip_file: nil, admin_set_id: nil) # rubocop:disable
row_errors = validator_context[:errors]
has_errors = missing_required.any? || headers.blank? || csv_data.empty? ||
file_validator.missing_files.any? || row_errors.any?
has_warnings = unrecognized.any? || file_validator.possible_missing_files?
has_warnings = unrecognized.any? || empty_columns.any? || file_validator.possible_missing_files?

result = {
headers: headers,
missingRequired: missing_required,
unrecognized: unrecognized,
emptyColumns: empty_columns,
rowCount: csv_data.length,
isValid: !has_errors,
hasWarnings: has_warnings,
Expand Down Expand Up @@ -177,10 +179,18 @@ def find_missing_required_headers(headers, field_metadata, mapping_manager)
def find_unrecognized_validation_headers(headers, valid_headers)
checker = DidYouMean::SpellChecker.new(dictionary: valid_headers)
headers
.reject { |h| valid_headers.include?(h) || valid_headers.include?(h.sub(/_\d+\z/, '')) }
.reject { |h| h.blank? || valid_headers.include?(h) || valid_headers.include?(h.sub(/_\d+\z/, '')) }
.index_with { |h| checker.correct(h).first }
end

def find_empty_column_positions(headers, raw_csv)
headers.each_with_index.filter_map do |h, i|
next if h.present?
has_data = raw_csv.any? { |row| row.fields[i].present? }
i + 1 if has_data
end
end

def resolve_parent_split_pattern(mappings)
split_val = mappings.dig('parents', 'split') || mappings.dig(:parents, :split)
return nil if split_val.blank?
Expand Down
19 changes: 12 additions & 7 deletions app/services/bulkrax/stepper_response_formatter.rb
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ def already_formatted?
def build_messages
issues = []
issues << missing_required_issue if @data[:missingRequired]&.any?
issues << unrecognized_fields_issue if @data[:unrecognized]&.any?
issues << unrecognized_fields_issue if @data[:unrecognized]&.any? || @data[:emptyColumns]&.any?
issues << file_references_issue if @data[:fileReferences]&.positive?
issues << row_errors_issue if @data[:rowErrors]&.any?

Expand All @@ -134,7 +134,7 @@ def build_messages
# @return [Hash] Validation status with severity, icon, title, summary, details
def validation_status
severity, icon, title = determine_severity_level
recognized = @data[:headers] - (@data[:unrecognized].keys || [])
recognized = @data[:headers].reject(&:blank?) - (@data[:unrecognized].keys || [])

{
severity: severity,
Expand Down Expand Up @@ -206,22 +206,27 @@ def missing_required_issue
#
# @return [Hash] Unrecognized fields issue structure
def unrecognized_fields_issue
all_items = unrecognized_fields_issue_items
{
type: 'unrecognized_fields',
severity: 'warning',
icon: 'fa-exclamation-triangle',
title: I18n.t('bulkrax.importer.guided_import.validation.unrecognized_title'),
count: @data[:unrecognized].length,
count: all_items.length,
description: I18n.t('bulkrax.importer.guided_import.validation.unrecognized_desc'),
items: unrecognized_fields_issue_items,
items: all_items,
defaultOpen: false
}
end

def unrecognized_fields_issue_items
@data[:unrecognized].partition(&:last)
.flatten(1)
.map { |field| { field: field.first, message: field.last ? I18n.t('bulkrax.importer.guided_import.validation.did_you_mean', suggestion: field.last) : nil } }
named = (@data[:unrecognized] || {}).partition(&:last)
.flatten(1)
.map { |field| { field: field.first, message: field.last ? I18n.t('bulkrax.importer.guided_import.validation.did_you_mean', suggestion: field.last) : nil } }
empty = (@data[:emptyColumns] || []).map do |col|
{ field: I18n.t('bulkrax.importer.guided_import.validation.empty_column', column: col), message: nil }
end
named + empty
end

# Format file references issue
Expand Down
1 change: 1 addition & 0 deletions config/locales/bulkrax.en.yml
Original file line number Diff line number Diff line change
Expand Up @@ -348,6 +348,7 @@ en:
message: "Field '%{field}' is required but is empty for this row."
suggestion: "Add a value for '%{field}'."
unable_to_process: Unable to process files for validation
empty_column: 'Column %{column} (no header)'
unrecognized_desc: 'These columns will be ignored during import:'
unrecognized_title: Unrecognized Fields
validations:
Expand Down
Loading