migrate_fmsl

#!/usr/bin/env ruby
# frozen_string_literal: true

# Script inputs a FMSL and may (depending on options)
# perform one or all of the following:
#
# - Migrate FMSL to a Fulcrum metadata.
# - Validate Fulcrum metadata, optionally checking for
#   the existence of resource files.
# - Generate a Fulcrum metadata CSV file which optionally
#   may include a column detailing any issues encountered
#   during validation.
# - Generate an XML representation of the Fulcrum metadta
#   and save it in a file.

# Determine the root directory of the code base.
script_dir = File.expand_path(File.dirname(__FILE__))
root_dir = File.dirname(script_dir)

require_relative File.join(root_dir, "lib", "logger")

script_logger = UMPTG::Logger.create(logger_fp: STDOUT)

require 'optparse'
require 'ostruct'

# Process the script parameters.
options = OpenStruct.new
options.fulcrum_headers_only = false
options.output_csv = false
options.output_xml = false
options.skip_resources_check = false
options.include_notes = false
option_parser = OptionParser.new do |opts|
  opts.banner = "Usage: #{File.basename(__FILE__)} [-c] [-f] [-n] [-r] [-x] fmsl_file [fmsl_file...]"
  opts.on('-c', '--csv', 'Save CSV file') do |output_csv|
    options.output_csv = true
  end
  opts.on('-f', '--fulcrum_headers', 'Include Fulcrum headers only') do |f|
    options.fulcrum_headers_only = true
  end
  opts.on('-n', '--notes', 'Include Validation Notes column in CSV') do |include_notes|
    options.include_notes = true
  end
  opts.on('-r', '--resources', '') do |skip_resources_check|
    options.skip_resources_check = true
  end
  opts.on('-x', '--xml', 'Save XML instance file') do |output_xml|
    options.output_xml = true
  end
  opts.on_tail('-h', '--help', 'Print this help message') do
    script_logger.info(opts)
    exit 0
  end
end
option_parser.parse!(ARGV)
if ARGV.count < 1
  script_logger.info(option_parser.help)
  return
end

fmsl_file_list = ARGV

require 'csv'

require_relative File.join(root_dir, "lib", "fmsl")
require_relative File.join(root_dir, "lib", "fulcrum", "manifest")

NOTES_COLUMN = "Validation Notes"

# Process each FMSL file specified.
fmsl_file_list.each do |fmsl_file|
  # Determine file existence.
  fmsl_file = File.expand_path(fmsl_file)
  if !File.file?(fmsl_file)
    script_logger.error("invalid FMSL file #{fmsl_file}")
    next
  end
  if options.output_csv
    script_logger.info("migrating FMSL #{File.basename(fmsl_file)}")
  else
    script_logger.info("validating FMSL #{File.basename(fmsl_file)}") unless options.output_csv
  end

  # Load the FMSL and migrate it to a Fulcrum metadata CSV.
  fmsl_body = UMPTG::FMSL.load(fmsl_file: fmsl_file)
  manifest_csv = UMPTG::FMSL.to_manifest(fmsl_body: fmsl_body)

  # Make sure there no duplicate columns.
  filter_headers = manifest_csv.headers.select {|h| !(h.nil? or h.empty?) }
  unless filter_headers.count == filter_headers.uniq.count
    #filter_headers.each {|h| puts h }
    # Tried Array#difference, but didn't work. ?
    sorted = filter_headers.sort
    df = []
    sorted.each_index do |ndx|
      next if ndx == 0
      df << sorted[ndx] if sorted[ndx] == sorted[ndx-1]
    end
    script_logger.error("FMSL #{File.basename(fmsl_file)} has duplicate columns #{df}.")
    next
  end

  # Convert the Fulcrum metadata CSV to an XML representation.
  manifest_name = File.join(File.basename(fmsl_file, ".*") + "_fulcrum")
  manifest = UMPTG::Fulcrum::Manifest::Document.new(
        :name => manifest_name,
        :csv_body => manifest_csv.to_s
      )
  collection = UMPTG::Fulcrum::Manifest::Validation::Collection.new
  collection.add_manifest(manifest)

  # If requested, write out the XML representation to a file.
  if options.output_xml
    output_file = File.join(File.dirname(fmsl_file), File.basename(fmsl_file, ".*") + ".xml")
    File.write(output_file, collection.xml_markup)
  end

  # Validate the Fulcrum metadata.
  error_count = 0
  resource_errors = {}

  # Check to see if resource files specified exists.
  if options.skip_resources_check
    script_logger.info("No resources check performed.")
  else
    manifest_csv.each do |row|
      next unless row['External Resource URL'].nil? or row['External Resource URL'].strip.empty?

      resource_name = row['File Name']
      if resource_name.nil? or resource_name.strip.empty?
        script_logger.warn("found row with no \"File Name\" value.")
        next
      end
      next if UMPTG::Fulcrum::Manifest.blank_row_name?(resource_name)
      next if resource_name.strip == UMPTG::Fulcrum::Manifest.MONOGRAPH_FILE_NAME

      # Determine if the resource file exists and is not 0 size.
      resource_file = File.join(File.dirname(fmsl_file), resource_name)
      if File.file?(resource_file)
        if File.size(resource_file) == 0
          error_count += 1
          resource_errors[resource_name] = [ "resource file is 0 size." ]
        end
      else
        error_count += 1
        resource_errors[resource_name] = [ "resource file does not exist." ]
      end
    end
    script_logger.info("No resources errors found.") if error_count == 0
  end

  # Validate the XML representation to determine any metadata field issues.
  # the the Fulcrum metadata.
  error_list = UMPTG::Fulcrum::Manifest::Validation::CollectionSchema.validate(:xml_markup => collection.xml_markup)
  script_logger.info("Error count: #{error_count + error_list.count}")

  # Display any resource existence issues.
  if error_count > 0
    resource_errors.each do |key,val|
      script_logger.error("#{key}: #{val.first}")
    end
  end

  # Display Fulcrum metadata field issues.
  if error_list.count > 0
    vtree = UMPTG::Fulcrum::Manifest::ValidationResult::VTreeBuilder.build(collection.xml_markup)
    script_logger.warn("vtree is nil") if vtree.nil?

    # Organize any issues within a list associated with the resource.
    error_list.each do |error|
      property = vtree.property(error.line) unless vtree.nil?
      
      unless options.include_notes
        level = error.error? ? "error" : error.warning? ? "warning" : error.fatal? ? "fatal" : "none"
        err_msg_list = [ "line: #{error.line}" ]
        err_msg_list << "\t#{property}" unless property.nil?
        err_msg_list << "\t#{property.resource_name}" unless property.nil?
        err_msg_list << "\t#{error}"
        err_msg = err_msg_list.join("\n")

        case
        when error.error?
          script_logger.error(err_msg)
        when error.warning?
          script_logger.warn(err_msg)
        when error.fatal?
          script_logger.fatal(err_msg)
        else
          script_logger.info(err_msg)
        end
      end

      resource_name = property.resource_name unless property.nil?
      unless resource_name.nil?
        if resource_errors[resource_name].nil?
          resource_errors[resource_name] = [ error ]
        else
          resource_errors[resource_name] << error
        end
      end
    end
  end

  # If requested, save the Fulcrum manifest file.
  if options.output_csv
    fheaders = options.fulcrum_headers_only ? \
              UMPTG::Fulcrum::Manifest::Validation::CollectionSchema.HEADERS : \
              manifest_csv.headers
    fulcrum_headers = options.include_notes ? \
              [ NOTES_COLUMN ] + fheaders : fheaders
    fulcrum_file = File.join(File.dirname(fmsl_file), manifest_name + ".csv")
    CSV.open(
            fulcrum_file,
            "w",
            :write_headers=> true,
            :force_quotes => true,
            :headers => fulcrum_headers
          ) do |csv|
      manifest_csv.each do |row|
        resource_name = row['File Name']
        error_list = resource_errors[resource_name]
        error_value = error_list.nil? ? "None" : error_list.join("\n")

        r = options.include_notes ? \
                { NOTES_COLUMN => error_value } : \
                {}
        row.each do |key,val|
          r[key] = val
        end
        csv << r
      end
    end
    script_logger.info("Saved CSV file #{fulcrum_file}.")
  end
end