CLDC-3802 sales bulk upload update 2025 (#2945)

* add 2025 sales bulk upload parser * lint and fix typos * fix typos, field numbers, staircasing tests * order field_11 values * test log creator selects correct year's parser * fix log to csv helper field order * update factory so test file fully succeeds * add 2025 BU test file method * apply new csv syntax * lint * lint * CLDC-3893 update property information field order * commonise prepare your file page * also update prep file page for lettings * CLDC-3893 update test * lint * don't error on blank discount if not RTB --------- Co-authored-by: Carolyn <[email protected]>
communitiesuk · Feb 27, 2025 · e5d10e2 · e5d10e2
1 parent fe89619
commit e5d10e2
Show file tree

Hide file tree

Showing 20 changed files with 4,137 additions and 168 deletions.
diff --git a/.rake_tasks~ b/.rake_tasks~
diff --git a/app/controllers/test_data_controller.rb b/app/controllers/test_data_controller.rb
@@ -47,22 +47,24 @@ def create_setup_test_sales_log
     redirect_to sales_log_path(log)
   end
 
-  def create_2024_test_sales_bulk_upload
-    return render_not_found unless FeatureToggle.create_test_logs_enabled?
-
-    file = Tempfile.new("test_sales_log.csv")
+  [2024, 2025].each do |year|
+    define_method("create_#{year}_test_sales_bulk_upload") do
+      return render_not_found unless FeatureToggle.create_test_logs_enabled?
 
-    log = FactoryBot.create(:sales_log, :completed, assigned_to: current_user, value: 180_000, deposit: 150_000)
-    log_to_csv = BulkUpload::SalesLogToCsv.new(log:, line_ending: "\n", overrides: { organisation_id: "ORG#{log.owning_organisation_id}", managing_organisation_id: "ORG#{log.owning_organisation_id}" })
-    file.write(log_to_csv.default_field_numbers_row)
-    file.write(log_to_csv.to_csv_row)
-    file.rewind
-    send_file file.path, type: "text/csv",
-                         filename: "test_sales_log.csv",
-                         disposition: "attachment",
-                         after_send: lambda {
-                                       file.close
-                                       file.unlink
-                                     }
+      file = Tempfile.new("#{year}_test_sales_log.csv")
+      log = FactoryBot.create(:sales_log, :completed, assigned_to: current_user, value: 180_000, deposit: 150_000, county: "Somerset", saledate: Time.zone.local(year.to_i, rand(4..12), rand(1..28)))
+      log_to_csv = BulkUpload::SalesLogToCsv.new(log:, line_ending: "\n", overrides: { organisation_id: "ORG#{log.owning_organisation_id}", managing_organisation_id: "ORG#{log.owning_organisation_id}" })
+      file.write(log_to_csv.default_field_numbers_row)
+      file.write(log_to_csv.to_csv_row)
+      file.rewind
+      send_file file.path,
+                type: "text/csv",
+                filename: "#{year}_test_sales_log.csv",
+                disposition: "attachment",
+                after_send: lambda {
+                  file.close
+                  file.unlink
+                }
+    end
   end
 end
diff --git a/app/helpers/bulk_upload/sales_log_to_csv.rb b/app/helpers/bulk_upload/sales_log_to_csv.rb
@@ -19,7 +19,7 @@ def to_csv_row(seed: nil)
     case year
     when 2022
       to_2022_csv_row
-    when 2023, 2024
+    when 2023, 2024, 2025
       to_year_csv_row(year, seed:)
     else
       raise NotImplementedError "No mapping function implemented for year #{year}"
@@ -67,6 +67,8 @@ def default_field_numbers_for_year(year)
       [6, 3, 4, 5, nil, 28, 30, 38, 47, 51, 55, 59, 31, 39, 48, 52, 56, 60, 37, 46, 50, 54, 58, 35, 43, 49, 53, 57, 61, 32, 33, 78, 80, 79, 81, 83, 84, nil, 62, 66, 64, 65, 63, 67, 69, 70, 68, 76, 77, 16, 17, 18, 26, 24, 25, 27, 8, 91, 95, 96, 97, 92, 93, 94, 98, 100, 101, 103, 104, 106, 110, 111, 112, 113, 114, 9, 116, 117, 118, 120, 124, 125, 126, 10, 11, nil, 127, 129, 133, 134, 135, 1, 2, nil, 73, nil, 75, 107, 108, 121, 122, 130, 131, 82, 109, 123, 132, 115, 15, 86, 87, 29, 7, 12, 13, 14, 36, 44, 45, 88, 89, 102, 105, 119, 128, 19, 20, 21, 22, 23, 34, 40, 41, 42, 71, 72, 74, 85, 90, 99]
     when 2024
       (1..131).to_a
+    when 2025
+      (1..121).to_a
     else
       raise NotImplementedError "No mapping function implemented for year #{year}"
     end
@@ -395,6 +397,141 @@ def to_2024_row
     ]
   end
 
+  def to_2025_row
+    [
+      log.saledate&.day,
+      log.saledate&.month,
+      log.saledate&.strftime("%y"),
+      overrides[:organisation_id] || log.owning_organisation&.old_visible_id,
+      overrides[:managing_organisation_id] || log.managing_organisation&.old_visible_id,
+      log.assigned_to&.email,
+      log.purchid,
+      log.ownershipsch,
+      log.ownershipsch == 1 ? log.type : "", # field_9: "What is the type of shared ownership sale?",
+      log.staircase, # 10
+      log.ownershipsch == 2 ? log.type : "", # field_11: "What is the type of discounted ownership sale?",
+      log.jointpur,
+      log.jointmore,
+      log.noint,
+      log.privacynotice,
+
+      log.uprn,
+      log.address_line1&.tr(",", " "), # 20
+      log.address_line2&.tr(",", " "),
+      log.town_or_city&.tr(",", " "),
+      log.county&.tr(",", " "),
+      ((log.postcode_full || "").split(" ") || [""]).first,
+      ((log.postcode_full || "").split(" ") || [""]).last,
+      log.la,
+      log.proptype,
+      log.beds,
+      log.builtype,
+      log.wchair,
+
+      log.age1,
+      log.sex1,
+      log.ethnic, # 30
+      log.nationality_all_group,
+      log.ecstat1,
+      log.buy1livein,
+      log.relat2,
+      log.age2,
+      log.sex2,
+      log.ethnic_group2,
+      log.nationality_all_buyer2_group,
+      log.ecstat2,
+      log.buy2livein, # 40
+      log.hholdcount,
+
+      log.relat3,
+      log.age3,
+      log.sex3,
+      log.ecstat3,
+      log.relat4,
+      log.age4,
+      log.sex4,
+      log.ecstat4,
+      log.relat5, # 50
+      log.age5,
+      log.sex5,
+      log.ecstat5,
+      log.relat6,
+      log.age6,
+      log.sex6,
+      log.ecstat6,
+
+      log.prevten,
+      log.ppcodenk,
+      ((log.ppostcode_full || "").split(" ") || [""]).first, # 60
+      ((log.ppostcode_full || "").split(" ") || [""]).last,
+      log.prevloc,
+      log.buy2living,
+      log.prevtenbuy2,
+
+      log.hhregres,
+      log.hhregresstill,
+      log.armedforcesspouse,
+      log.disabled,
+      log.wheel,
+
+      log.income1, # 70
+      log.inc1mort,
+      log.income2,
+      log.inc2mort,
+      log.hb,
+      log.savings.present? || "R",
+      log.prevown,
+      log.prevshared,
+
+      log.resale,
+      log.proplen,
+      log.hodate&.day, # 80
+      log.hodate&.month,
+      log.hodate&.strftime("%y"),
+      log.frombeds,
+      log.fromprop,
+      log.socprevten,
+      log.value,
+      log.equity,
+      log.mortgageused,
+      log.mortgage,
+      log.mortlen, # 90
+      log.deposit,
+      log.cashdis,
+      log.mrent,
+      log.mscharge,
+      log.management_fee,
+
+      log.stairbought,
+      log.stairowned,
+      log.staircasesale,
+      log.firststair,
+      log.initialpurchase&.day, # 100
+      log.initialpurchase&.month,
+      log.initialpurchase&.strftime("%y"),
+      log.numstair,
+      log.lasttransaction&.day,
+      log.lasttransaction&.month,
+      log.lasttransaction&.strftime("%y"),
+      log.value,
+      log.equity,
+      log.mortgageused,
+      log.mrentprestaircasing, # 110
+      log.mrent,
+
+      log.proplen,
+      log.value,
+      log.grant,
+      log.discount,
+      log.mortgageused,
+      log.mortgage,
+      log.mortlen,
+      log.extrabor,
+      log.deposit, # 120
+      log.mscharge,
+    ]
+  end
+
   def custom_field_numbers_row(seed: nil, field_numbers: nil)
     if seed
       ["Field number"] + field_numbers.shuffle(random: Random.new(seed))

diff --git a/app/models/bulk_upload.rb b/app/models/bulk_upload.rb
@@ -104,6 +104,8 @@ def prefix_namespace
                  end
 
     year_class = case year
+                 when 2025
+                   "Year2025"
                  when 2024
                    "Year2024"
                  when 2023

diff --git a/app/models/forms/bulk_upload_form/prepare_your_file.rb b/app/models/forms/bulk_upload_form/prepare_your_file.rb
@@ -10,10 +10,7 @@ class PrepareYourFile
       attribute :organisation_id, :integer
 
       def view_path
-        case year
-        when 2024
-          "bulk_upload_#{log_type}_logs/forms/prepare_your_file_2024"
-        end
+        "bulk_upload_#{log_type}_logs/forms/prepare_your_file"
       end
 
       def back_path
@@ -42,6 +39,10 @@ def year_combo
         "#{year} to #{year + 1}"
       end
 
+      def slash_year_combo
+        "#{year}/#{(year + 1) % 100}"
+      end
+
       def save!
         true
       end

diff --git a/app/services/bulk_upload/sales/log_creator.rb b/app/services/bulk_upload/sales/log_creator.rb
@@ -33,6 +33,8 @@ def csv_parser
                       BulkUpload::Sales::Year2023::CsvParser.new(path:)
                     when 2024
                       BulkUpload::Sales::Year2024::CsvParser.new(path:)
+                    when 2025
+                      BulkUpload::Sales::Year2025::CsvParser.new(path:)
                     else
                       raise "csv parser not found"
                     end

diff --git a/app/services/bulk_upload/sales/validator.rb b/app/services/bulk_upload/sales/validator.rb
@@ -108,6 +108,8 @@ def csv_parser
                       BulkUpload::Sales::Year2023::CsvParser.new(path:)
                     when 2024
                       BulkUpload::Sales::Year2024::CsvParser.new(path:)
+                    when 2025
+                      BulkUpload::Sales::Year2025::CsvParser.new(path:)
                     else
                       raise "csv parser not found"
                     end

diff --git a/app/services/bulk_upload/sales/year2024/row_parser.rb b/app/services/bulk_upload/sales/year2024/row_parser.rb
@@ -375,6 +375,7 @@ class BulkUpload::Sales::Year2024::RowParser
               greater_than_or_equal_to: 0,
               less_than_or_equal_to: 70,
               if: :discounted_ownership?,
+              allow_blank: true,
             },
             on: :before_log
 

diff --git a/app/services/bulk_upload/sales/year2025/csv_parser.rb b/app/services/bulk_upload/sales/year2025/csv_parser.rb
@@ -0,0 +1,124 @@
+require "csv"
+
+class BulkUpload::Sales::Year2025::CsvParser
+  include CollectionTimeHelper
+
+  FIELDS = 121
+  MAX_COLUMNS = 142
+  FORM_YEAR = 2025
+
+  attr_reader :path
+
+  def initialize(path:)
+    @path = path
+  end
+
+  def row_offset
+    if with_headers?
+      rows.find_index { |row| row[0].present? && row[0].match(/field number/i) } + 1
+    else
+      0
+    end
+  end
+
+  def col_offset
+    with_headers? ? 1 : 0
+  end
+
+  def cols
+    @cols ||= ("A".."DR").to_a
+  end
+
+  def row_parsers
+    @row_parsers ||= body_rows.map { |row|
+      next if row.empty?
+
+      stripped_row = row[col_offset..]
+      hash = Hash[field_numbers.zip(stripped_row)]
+
+      BulkUpload::Sales::Year2025::RowParser.new(hash)
+    }.compact
+  end
+
+  def body_rows
+    rows[row_offset..]
+  end
+
+  def rows
+    @rows ||= CSV.parse(normalised_string, row_sep:)
+  end
+
+  def column_for_field(field)
+    cols[field_numbers.find_index(field) + col_offset]
+  end
+
+  def wrong_template_for_year?
+    collection_start_year_for_date(first_record_start_date) != FORM_YEAR
+  rescue Date::Error
+    false
+  end
+
+  def missing_required_headers?
+    !with_headers?
+  end
+
+  def correct_field_count?
+    valid_field_numbers_count = field_numbers.count { |f| f != "field_blank" }
+
+    valid_field_numbers_count == FIELDS
+  end
+
+private
+
+  def default_field_numbers
+    (1..FIELDS).map do |number|
+      if number.to_s.match?(/^[0-9]+$/)
+        "field_#{number}"
+      else
+        "field_blank"
+      end
+    end
+  end
+
+  def field_numbers
+    @field_numbers ||= if with_headers?
+                         rows[row_offset - 1][col_offset..].map { |number| number.to_s.match?(/^[0-9]+$/) ? "field_#{number}" : "field_blank" }
+                       else
+                         default_field_numbers
+                       end
+  end
+
+  def headers
+    @headers ||= ("field_1".."field_#{FIELDS}").to_a
+  end
+
+  def with_headers?
+    # we will eventually want to validate that headers exist for this year
+    rows.map { |r| r[0] }.any? { |cell| cell&.match?(/field number/i) }
+  end
+
+  def row_sep
+    "\n"
+  end
+
+  def normalised_string
+    return @normalised_string if @normalised_string
+
+    @normalised_string = File.read(path, encoding: "bom|utf-8")
+    @normalised_string.gsub!("\r\n", "\n")
+    @normalised_string.scrub!("")
+    @normalised_string.tr!("\r", "\n")
+
+    @normalised_string
+  end
+
+  def first_record_start_date
+    if with_headers?
+      year = row_parsers.first.field_3.to_s.strip.length.between?(1, 2) ? row_parsers.first.field_3.to_i + 2000 : row_parsers.first.field_3.to_i
+      Date.new(year, row_parsers.first.field_2.to_i, row_parsers.first.field_1.to_i)
+    else
+      year = rows.first[2].to_s.strip.length.between?(1, 2) ? rows.first[2].to_i + 2000 : rows.first[2].to_i
+      Date.new(year, rows.first[1].to_i, rows.first[0].to_i)
+    end
+  end
+end