diff --git a/lib/nokodown.rb b/lib/nokodown.rb index fadf7f7..ac4d1e3 100644 --- a/lib/nokodown.rb +++ b/lib/nokodown.rb @@ -7,6 +7,7 @@ require_relative "nokodown/dom" require_relative "nokodown/stats" require_relative "nokodown/patterns" +require_relative "nokodown/heading_anchors" require_relative "nokodown/tables" require_relative "nokodown/url_resolver" require_relative "nokodown/metadata" diff --git a/lib/nokodown/dom.rb b/lib/nokodown/dom.rb index d41f102..d63fa72 100644 --- a/lib/nokodown/dom.rb +++ b/lib/nokodown/dom.rb @@ -36,6 +36,9 @@ module Dom # survives the copy that article cleaning works on. THREAD = "data-nokodown-thread" + # Set by HeadingAnchors on elements whose id is made from their heading. + ANCHOR = "data-nokodown-anchor" + module_function def assign_keys(root) @@ -109,12 +112,17 @@ def squish(string) def signature(node) classes = node["class"] - id = node["id"] + id = name_id(node) return "" if classes.nil? && id.nil? signature_of("#{classes} #{id}") end + # The id as a name for the element; a heading anchor only repeats the heading. + def name_id(node) + node.key?(ANCHOR) ? nil : node["id"] + end + def signature_of(value) # Generated ids (Webflow "w-node-") would otherwise read as words. value = value.gsub(UUID, " ") if value.match?(UUID) diff --git a/lib/nokodown/heading_anchors.rb b/lib/nokodown/heading_anchors.rb new file mode 100644 index 0000000..68b6b6d --- /dev/null +++ b/lib/nokodown/heading_anchors.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Nokodown + # Sites give headings ids made from their text so tables of contents can + # link to them ("h-what-is-social-media", "related-coverage-2"), and + # documentation generators put the same id on the section the heading + # opens. Such an id says what the section is about, not what kind of + # section it is: "social" in it does not make a share bar. Marked once, so + # every class/id rule can leave it out. + module HeadingAnchors + PREFIX = /\A[a-z][-_]/i.freeze + COUNTER = /[-_]\d+\z/.freeze + + # Headings that are nothing but the name of a widget: "Feedback", "Table + # of contents". Reviews are only removed from product pages, where a + # section headed by them is the reviews widget, so naming them anywhere + # will do: "Ratings and reviews". + WIDGET_NAME = /\A(?:#{Patterns::PAGE_FURNITURE}|#{Patterns::SECONDARY})\z|#{Patterns::REVIEWS}/.freeze + + module_function + + def mark(root) + root.css("[id]").each { |node| node[Dom::ANCHOR] = "1" if anchor?(node) } + end + + def anchor?(node) + label = label(node) + return false if label.nil? || junk?(label) + + text = compact(label) + !text.empty? && variants(node["id"]).any? { |variant| compact(variant) == text } + end + + # A junk label's anchor names the widget under it, which goes too; keeping + # the heading would leave it without a section. Only its words count: the + # permalink the anchor markup adds ("¶", "#", a zero-width space) would + # keep the anchored patterns from matching. + def junk?(label) + words = label.downcase.scan(/[\p{L}\p{N}]+/) + phrase = words.join(" ") + return true if phrase.match?(Patterns::NOISE_HEADING) || phrase.match?(Patterns::BOILERPLATE_TEXT) + + words.join("-").match?(WIDGET_NAME) + end + + # The heading itself, a wiki's headline span inside it, or the heading + # that opens a section (after any empty anchor targets). + def label(node) + return Dom.text(node) if Dom.heading?(node) || Dom.heading?(node.parent) + + heading = opening_heading(node) + heading && Dom.text(heading) + end + + def opening_heading(node) + child = node.first_element_child + child = child.next_element while child && !Dom.heading?(child) && empty?(child) + child if child && Dom.heading?(child) + end + + def empty?(node) + node.first_element_child.nil? && node.content.strip.empty? + end + + # A one-letter prefix ("h-", "s-") and a counter for repeated headings. + def variants(id) + [id, id.sub(PREFIX, "")].flat_map { |variant| [variant, variant.sub(COUNTER, "")] } + end + + # Slugifiers differ in how they treat case, accents, apostrophes and spaces. + def compact(string) + string.unicode_normalize(:nfkd).downcase.gsub(/[^\p{L}\p{N}]/, "") + end + end +end diff --git a/lib/nokodown/preprocessor.rb b/lib/nokodown/preprocessor.rb index 6d7655e..284f7a2 100644 --- a/lib/nokodown/preprocessor.rb +++ b/lib/nokodown/preprocessor.rb @@ -52,6 +52,7 @@ def call @body.css("pre [class], table[class] td[class]").each { |node| node.remove if node["class"].downcase.match?(LINE_NUMBERS) } remove_hidden remove_form_controls + HeadingAnchors.mark(@body) Dom.assign_keys(@body) end diff --git a/lib/nokodown/scorer.rb b/lib/nokodown/scorer.rb index 1f0cd6a..d204f00 100644 --- a/lib/nokodown/scorer.rb +++ b/lib/nokodown/scorer.rb @@ -17,10 +17,11 @@ def self.class_weight(node) end # Conditional cleaning deletes whatever weighs negative, so it ignores CSS - # utility tokens. Candidate scoring keeps Readability's weight, which some - # pages need to pick the right candidate. + # utility tokens and heading anchors ("media" in "h-social-media-tips"). + # Candidate scoring keeps Readability's weight, which some pages need to + # pick the right candidate. def self.cleaning_weight(node) - [node["class"], node["id"]].sum do |value| + [node["class"], Dom.name_id(node)].sum do |value| tokens = value.to_s.downcase.split.reject { |token| utility_class?(token) } words_weight(tokens.join(" ")) end diff --git a/spec/nokodown/boilerplate_spec.rb b/spec/nokodown/boilerplate_spec.rb index 479fcfe..76515ad 100644 --- a/spec/nokodown/boilerplate_spec.rb +++ b/spec/nokodown/boilerplate_spec.rb @@ -84,6 +84,38 @@ def prose(topic, count) end end + describe "ids made from a heading's text" do + let(:article) { "

Selling on social media

#{prose('Social selling', 4)}" } + + it "keeps headings and sections whose id is their heading's anchor" do + html = <<~HTML + #{article} +

What is social shopping?

#{prose('Social shopping', 1)} +

Sponsorship deals

#{prose('Sponsorship', 1)} +

Cookies

#{prose('Reading cookies', 1)}
+ HTML + + text = remaining(html) + text.should include("What is social shopping?") + text.should include("Sponsorship deals") + text.should include("Reading cookies is explained") + end + + it "still removes junk labels, ids that are not anchors and furniture classes" do + html = <<~HTML + #{article} + +

Share this article

Copy link
+ + +

Social media tips

+ HTML + + text = remaining(html) + %w[Related Share Older Follow tips].each { |word| text.should_not include(word) } + end + end + describe "repeated modules" do def cards(classes, count) Array.new(count) do |index| diff --git a/spec/nokodown/heading_anchors_spec.rb b/spec/nokodown/heading_anchors_spec.rb new file mode 100644 index 0000000..0440fa9 --- /dev/null +++ b/spec/nokodown/heading_anchors_spec.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +RSpec.describe Nokodown::HeadingAnchors do + def anchor?(html) + body = Nokolexbor::HTML("#{html}").at_css("body") + described_class.anchor?(body.at_css("[id]")) + end + + it "recognizes ids made from a heading's text, with a one-letter prefix or a counter" do + anchor?('

What’s Next for Social Shopping?

').should == true + anchor?('').should == true + anchor?('

Login shell

').should == true + anchor?('

Social Tips

').should == true + anchor?('

Qué es el marketing social

').should == true + anchor?('

Related to Portage

').should == true + end + + it "recognizes the id of the section a heading opens" do + anchor?('

Cookies

Read them from the request.

').should == true + anchor?('

Sharing models

Import them.

').should == true + end + + it "does not take other ids for anchors" do + anchor?('').should == false + anchor?('

Latest posts.

Social feed

').should == false + anchor?('Social sharing').should == false + end + + it "does not take the id of a junk label for an anchor, since it names the widget under it" do + anchor?('').should == false + anchor?('

Share this article

').should == false + anchor?('

Feedback

').should == false + anchor?('

Table of contents

').should == false + anchor?('

Customer Reviews

').should == false + anchor?('

Ratings and Reviews

Sturdy and light.

').should == false + end + + it "reads a junk label without the permalink that anchor markup adds to it" do + anchor?('

Comments

Jane wrote: thanks.

').should == false + anchor?('').should == false + anchor?('

# Share this article

').should == false + anchor?(%()).should == false + end +end diff --git a/spec/nokodown/product_reviews_spec.rb b/spec/nokodown/product_reviews_spec.rb index 8d52af6..09d2187 100644 --- a/spec/nokodown/product_reviews_spec.rb +++ b/spec/nokodown/product_reviews_spec.rb @@ -42,6 +42,15 @@ def markdown(head, main) result.should_not include("summer of surveys") end + it "removes a reviews section whose id is made from its heading" do + anchored = '

Ratings and Reviews

It survived a whole summer of surveys ' \ + "in my back pocket, and the paper takes pencil and fountain pen alike.

" + result = markdown(product_type, "#{description}#{details}#{anchored}") + + result.should include("dot-grid paper") + result.should_not include("summer of surveys") + end + it "removes schema.org reviews even though each has a description of its own" do review = '

' \ "Bought three of these for my survey team and every one of them made it through a wet season of work.

" diff --git a/spec/nokodown_spec.rb b/spec/nokodown_spec.rb index 48e8e9f..eea5f6c 100644 --- a/spec/nokodown_spec.rb +++ b/spec/nokodown_spec.rb @@ -314,6 +314,19 @@ def story(extra) end end + it "keeps a heading whose anchor holds a negative class word" do + markdown = story('

Media coverage of tidal lagoons

Newspapers mostly covered the cost.

') + + markdown.should include("## Media coverage of tidal lagoons") + end + + it "still drops comments whose heading carries a permalink" do + markdown = story('

Comments

Jane wrote: the turbine part was the clearest.

') + + markdown.should_not include("Comments") + markdown.should_not include("Jane wrote") + end + it "still drops copies shown only at some screen sizes" do markdown = story('

Narrow screen copy of the intro.

')