Merge pull request #3730 from mlibrary/HELIO-4125/index_epub_accessibility_metadata

sethaj · web-flow · commit 0228c620affe · 2024-12-17T09:28:58.000-05:00
HELIO-4125 - index EPUB accessibility metadata
diff --git a/app/indexers/monograph_indexer.rb b/app/indexers/monograph_indexer.rb
@@ -86,6 +86,9 @@ def generate_solr_document # rubocop:disable Metrics/PerceivedComplexity, Metric
 
       # HELIO-2428 index the "full" doi url if there's a doi
       solr_doc['doi_url_ssim'] = "https://doi.org/" + object.doi if object.doi.present?
+
+      # HELIO-4125 - Extract EPUB metadata on ingest and store in Solr
+      maybe_index_accessibility_metadata(solr_doc)
     end
   end
 
@@ -170,4 +173,10 @@ def all_product_names_for_monograph(obj)
     all_product_names << "Unrestricted" if all_product_ids.include?(0)
     all_product_names + Greensub::Product.where(id: all_product_ids).map { |product| product.name || product.identifier }
   end
+
+  def maybe_index_accessibility_metadata(solr_doc)
+    epub_fr = FeaturedRepresentative.where(work_id: object.id, kind: 'epub')&.first
+    return if epub_fr.blank?
+    EpubAccessibilityMetadataIndexingService.index(epub_fr.file_set_id, solr_doc)
+  end
 end
diff --git a/app/services/epub_accessibility_metadata_indexing_service.rb b/app/services/epub_accessibility_metadata_indexing_service.rb
@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+class EpubAccessibilityMetadataIndexingService
+  class << self
+    # This will be called from MonographIndexer. Given that nothing indexed here is crucial to Fulcrum's core features,...
+    # no errors should be raised that would prevent the Monograph document from being created.
+    # For all of the fields here I'll use string (not English text), stored and indexed and use the expected cardinality.
+    # This should allow any/all to be used in Blacklight facets.
+    def index(epub_file_set_id, solr_doc)
+      root_path = UnpackService.root_path_from_noid(epub_file_set_id, 'epub')
+      return unless Dir.exist? root_path
+
+      container_file = File.join(root_path, "META-INF/container.xml")
+      return unless File.exist? container_file
+
+      container = Nokogiri::XML(File.open(container_file)).remove_namespaces!
+      container.xpath("//rootfile").length > 1 ? 'yes' : 'no'
+      # Accessibility metadata extraction requires one, and only one, rendition to be present.
+      return unless container.xpath("//rootfile").length == 1
+
+      content_file = container.xpath("//rootfile/@full-path").text
+      content_file = File.join(root_path, content_file)
+      return unless File.exist? content_file
+
+      content = Nokogiri::XML(File.open(content_file)).remove_namespaces!
+      package_element = content.at_css('package')
+      epub_version = package_element['version']&.strip if package_element.present?
+      return if epub_version.blank?
+
+      # Indexing this for no particular reason. Might be useful at some point.
+      solr_doc['epub_version_ssi'] = epub_version
+
+      # This should allow the relevant values to be detected for both EPUB 2 and EPUB 3, though only the latter should...
+      # be present on Fulcrum. See Daisy links below.
+      @epub_2 = epub_version.start_with?('2')
+      @meta_attribute = @epub_2 ? 'name' : 'property'
+
+      @content_metadata = content.at_css('metadata')
+      return if @content_metadata.blank?
+
+      # for these schema values see https://kb.daisy.org/publishing/docs/metadata/schema.org/
+      solr_doc['epub_a11y_accessibility_summary_ssi'] = accessibility_summary
+      solr_doc['epub_a11y_accessibility_features_ssim'] = accessibility_features
+      solr_doc['epub_a11y_access_mode_ssim'] = access_mode
+      @access_mode_sufficient = access_mode_sufficient
+      solr_doc['epub_a11y_access_mode_sufficient_ssim'] = @access_mode_sufficient
+
+      # for these evaluation values see https://kb.daisy.org/publishing/docs/metadata/evaluation.html
+      solr_doc['epub_a11y_conforms_to_ssi'] = conforms_to
+      solr_doc['epub_a11y_certified_by_ssi'] = certified_by
+      solr_doc['epub_a11y_certifier_credential_ssi'] = certifier_credential
+
+      # This is a derived value for convenience in Fulcrum UI use.
+      solr_doc['epub_a11y_screen_reader_friendly_ssi'] = screen_reader_friendly
+    end
+
+    def accessibility_summary
+      value = @content_metadata.at_css("meta[#{@meta_attribute}='schema:accessibilitySummary']")
+      return nil if value.blank?
+      @epub_2 ? value['content']&.strip : value.text&.strip
+    end
+
+    def accessibility_features
+      # this involves multiple entries in separate meta tags
+      values = @content_metadata.css("meta[#{@meta_attribute}='schema:accessibilityFeature']")
+
+      values = if @epub_2
+                 values&.map { |value| value['content']&.strip }
+               else
+                 values&.map { |value| value&.text&.strip }
+               end
+      # want to ensure the indexer is set to nil not [] if these are not present, keeping the field off the doc entirely
+      values.presence
+    end
+
+    def access_mode
+      # this involves multiple entries in separate meta tags
+      values = @content_metadata.css("meta[#{@meta_attribute}='schema:accessMode']")
+
+      values = if @epub_2
+                 values&.map { |value| value['content']&.strip }
+               else
+                 values&.map { |value| value&.text&.strip }
+               end
+      # want to ensure the indexer is set to nil not [] if these are not present, keeping the field off the doc entirely
+      values.presence
+    end
+
+    def access_mode_sufficient
+      # this one has multiple entries in one value, comma separated
+      values = @content_metadata.at_css("meta[#{@meta_attribute}='schema:accessModeSufficient']")
+      return nil if values.blank?
+      values = @epub_2 ? values['content']&.split(',') : values.text&.split(',')
+      values&.reject(&:blank?)&.map(&:strip)
+    end
+
+    def conforms_to
+      value = @content_metadata.at_css("meta[#{@meta_attribute}='dcterms:conformsTo']")
+      return nil if value.blank?
+      @epub_2 ? value['content']&.strip : value.text&.strip
+    end
+
+    def certified_by
+      value = @content_metadata.at_css("meta[#{@meta_attribute}='a11y:certifiedBy']")
+      return nil if value.blank?
+      @epub_2 ? value['content']&.strip : value.text&.strip
+    end
+
+    def certifier_credential
+      value = @content_metadata.at_css("meta[#{@meta_attribute}='a11y:certifierCredential']")
+      return nil if value.blank?
+      @epub_2 ? value['content']&.strip : value.text&.strip
+    end
+
+    def screen_reader_friendly
+      if @access_mode_sufficient.present?
+        if @access_mode_sufficient.count == 1 && @access_mode_sufficient[0] == 'textual'
+          'true'
+        else
+          'false'
+        end
+      else
+        # I guess it's OK that this will always have a value even if all the other a11y metadata is missing.
+        'unknown'
+      end
+    end
+  end
+end
diff --git a/spec/indexers/monograph_indexer_spec.rb b/spec/indexers/monograph_indexer_spec.rb
@@ -85,14 +85,23 @@
       end
     end
 
-    context "ebook representative table of contents" do
+    context "ebook metadata indexed at the Monograph level" do
       before do
         create(:featured_representative, work_id: monograph.id, file_set_id: file_set.id, kind: "epub")
-        UnpackJob.perform_now(file_set.id, "epub") # to index the epub's table of contents HELIO-3870
+        UnpackJob.perform_now(file_set.id, "epub") # to index the epub's table of contents HELIO-3870 and a11y metadata HELIO-4125
       end
 
-      it "indexes the epub/pdf_ebook's ToC if there is one" do
-        expect(subject['table_of_contents_tesim']).to include("Chapter 73. Stubb and Flask Kill a Right Whale; and Then Have a Talk")
+      context 'ebook representative table of contents' do
+        it "indexes the epub/pdf_ebook's ToC if there is one" do
+          expect(subject['table_of_contents_tesim']).to include("Chapter 73. Stubb and Flask Kill a Right Whale; and Then Have a Talk")
+        end
+      end
+
+      context 'Runs EpubAccessibilityMetadataIndexingService' do
+        it 'indexes fields from the EPUB OPF "package" file on the Monograph' do
+          # checking that `version` was indexed wll suffice, EpubAccessibilityMetadataIndexingServiceSpec covers them all
+          expect(subject['epub_version_ssi']).to eq("3.0")
+        end
       end
     end
 
diff --git a/spec/jobs/unpack_job_spec.rb b/spec/jobs/unpack_job_spec.rb
@@ -24,9 +24,15 @@
         monograph.ordered_members << reflowable_epub
         monograph.save!
         reflowable_epub.save!
+        # the `find` and `parent` stubs are necessary for the `update_index` stub to work
+        allow(FileSet).to receive(:find).with(reflowable_epub.id).and_return(reflowable_epub)
+        allow(reflowable_epub).to receive(:parent).and_return(monograph)
+        allow(monograph).to receive(:update_index)
       end
 
       it "unzips the epub, caches the ToC, creates the search database and doesn't make chapter files derivatives" do
+        # check that we're reindexing to get the ToC and accessibility metadata on the parent Monograph's Solr doc
+        expect(monograph).to receive(:update_index)
         described_class.perform_now(reflowable_epub.id, 'epub')
         expect(JSON.parse(EbookTableOfContentsCache.find_by(noid: reflowable_epub.id).toc).length).to eq 3
         expect(JSON.parse(EbookTableOfContentsCache.find_by(noid: reflowable_epub.id).toc)[0]["title"]).to eq "Damage report!"
@@ -48,9 +54,15 @@
         monograph.ordered_members << fixed_layout_epub
         monograph.save!
         fixed_layout_epub.save!
+        # the `find` and `parent` stubs are necessary for the `update_index` stub to work
+        allow(FileSet).to receive(:find).with(fixed_layout_epub.id).and_return(fixed_layout_epub)
+        allow(fixed_layout_epub).to receive(:parent).and_return(monograph)
+        allow(monograph).to receive(:update_index)
       end
 
       it "unzips the epub, caches the ToC, creates the search database and makes the chapter files derivatives" do
+        # check that we're reindexing to get the ToC and accessibility metadata on the parent Monograph's Solr doc
+        expect(monograph).to receive(:update_index)
         described_class.perform_now(fixed_layout_epub.id, 'epub')
         expect(File.exist?(File.join(root_path, fixed_layout_epub.id + '.db'))).to be true
         expect(Dir.exist?(chapters_dir)).to be true
diff --git a/spec/services/epub_accessibility_metadata_indexing_service_spec.rb b/spec/services/epub_accessibility_metadata_indexing_service_spec.rb
