Merge pull request #124 from unepwcmc/feat/rake-task-to-correct-inval…

…id-data

feat: rake task to update records with invalid data (WIP)

CHANGELOG.md

@@ -0,0 +1,7 @@
+# 1.0.0
+
+Site has been released for a long time, but starting at 1.0.0 for the purposes of adding a change log
+
+- use date_discovery for date filtering instead of created_at
+- rake task to fix non-breaking spaces and  character in database
+- process strings during import to prevent the above issue

lib/modules/csv_converter.rb

@@ -23,6 +23,7 @@ def convert header, value
   end
 
   def add_genus value, type
+    remove_non_breaking_spaces(value)
     @report.data["genera"][type] ||= []
 
     # Call find_genus using the species name. 
@@ -35,6 +36,7 @@ def add_genus value, type
   end
 
   def answer question, value, page=nil
+    remove_non_breaking_spaces(value)
     if page
       @report.data["answers"][page] ||= [{}]
       @report.data["answers"][page][0][question] ||= {}
@@ -82,6 +84,7 @@ def find_user value
   end
 
   def find_genus value
+    remove_non_breaking_spaces(value)
     genera = {
       "Bonobo (Pan)" => "bonobo",
       "Chimpanzee (Pan)" => "chimpanzee",
@@ -183,4 +186,10 @@ def add_quantity value, ape_condition
   def self.columns
     CONVERSIONS.keys
   end
+
+  def remove_non_breaking_spaces(value)
+    # Some file uploads contain data that has non-breaking sapces instead of spaces,
+    # so replace them with spaces.
+    value.gsub!(/\u00a0/," ") if value.is_a?(String)
+  end
 end

lib/tasks/fix_invalid_data_from_imports.rake

@@ -0,0 +1,49 @@
+desc 'Iterate over bjson data structure and replace all non-breaking spaces with spaces and remove all  characters'
+task fix_invalid_data_from_imports: :environment do
+  Report.find_each do |report|
+    data = report.data
+    hash_iterator(data, report.id)
+    report.update(data: data)
+  end
+end
+
+def hash_iterator(hash, report_id)
+  keys = hash.keys
+  keys.each do |key|
+    value = hash[key]
+
+    if value.is_a?(Hash)
+      hash_iterator(value, report_id)
+    elsif value.is_a?(Array)
+      array_iterator(value, report_id)
+    elsif value.is_a?(String)
+      clean_string(value, report_id)
+    end
+  end
+end
+
+def array_iterator(value_array, report_id)
+  value_array.each do |value|
+    if value.is_a?(String)
+      clean_string(value, report_id)
+    elsif value.is_a?(Hash)
+      hash_iterator(value, report_id)
+    elsif value.is_a?(Array)
+      array_iterator(value, report_id)
+    end
+  end
+end
+
+def clean_string(value, report_id)
+  non_breaking_space = value =~ /\u00a0/
+  if non_breaking_space
+    puts "removing non-breaking space from #{value} on Report ##{report_id}"
+    value.gsub!(/\u00a0/," ")
+  end
+
+  invalid_character_index = value =~ /[Â]/
+  if invalid_character_index
+    puts "removing  from #{value} on Report ##{report_id}"
+    value.slice!(invalid_character_index)
+  end
+end