Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions lib/nokodown.rb
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
require_relative "nokodown/dom"
require_relative "nokodown/stats"
require_relative "nokodown/patterns"
require_relative "nokodown/heading_anchors"
require_relative "nokodown/tables"
require_relative "nokodown/url_resolver"
require_relative "nokodown/metadata"
Expand Down
10 changes: 9 additions & 1 deletion lib/nokodown/dom.rb
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@ module Dom
# survives the copy that article cleaning works on.
THREAD = "data-nokodown-thread"

# Set by HeadingAnchors on elements whose id is made from their heading.
ANCHOR = "data-nokodown-anchor"

module_function

def assign_keys(root)
Expand Down Expand Up @@ -109,12 +112,17 @@ def squish(string)

def signature(node)
classes = node["class"]
id = node["id"]
id = name_id(node)
return "" if classes.nil? && id.nil?

signature_of("#{classes} #{id}")
end

# The id as a name for the element; a heading anchor only repeats the heading.
def name_id(node)
node.key?(ANCHOR) ? nil : node["id"]
end

def signature_of(value)
# Generated ids (Webflow "w-node-<uuid>") would otherwise read as words.
value = value.gsub(UUID, " ") if value.match?(UUID)
Expand Down
75 changes: 75 additions & 0 deletions lib/nokodown/heading_anchors.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
# frozen_string_literal: true

module Nokodown
# Sites give headings ids made from their text so tables of contents can
# link to them ("h-what-is-social-media", "related-coverage-2"), and
# documentation generators put the same id on the section the heading
# opens. Such an id says what the section is about, not what kind of
# section it is: "social" in it does not make a share bar. Marked once, so
# every class/id rule can leave it out.
module HeadingAnchors
PREFIX = /\A[a-z][-_]/i.freeze
COUNTER = /[-_]\d+\z/.freeze

# Headings that are nothing but the name of a widget: "Feedback", "Table
# of contents". Reviews are only removed from product pages, where a
# section headed by them is the reviews widget, so naming them anywhere
# will do: "Ratings and reviews".
WIDGET_NAME = /\A(?:#{Patterns::PAGE_FURNITURE}|#{Patterns::SECONDARY})\z|#{Patterns::REVIEWS}/.freeze

module_function

def mark(root)
root.css("[id]").each { |node| node[Dom::ANCHOR] = "1" if anchor?(node) }
end

def anchor?(node)
label = label(node)
return false if label.nil? || junk?(label)

text = compact(label)
!text.empty? && variants(node["id"]).any? { |variant| compact(variant) == text }
end

# A junk label's anchor names the widget under it, which goes too; keeping
# the heading would leave it without a section. Only its words count: the
# permalink the anchor markup adds ("¶", "#", a zero-width space) would
# keep the anchored patterns from matching.
def junk?(label)
words = label.downcase.scan(/[\p{L}\p{N}]+/)
phrase = words.join(" ")
return true if phrase.match?(Patterns::NOISE_HEADING) || phrase.match?(Patterns::BOILERPLATE_TEXT)

words.join("-").match?(WIDGET_NAME)
end

# The heading itself, a wiki's headline span inside it, or the heading
# that opens a section (after any empty anchor targets).
def label(node)
return Dom.text(node) if Dom.heading?(node) || Dom.heading?(node.parent)

heading = opening_heading(node)
heading && Dom.text(heading)
end

def opening_heading(node)
child = node.first_element_child
child = child.next_element while child && !Dom.heading?(child) && empty?(child)
child if child && Dom.heading?(child)
end

def empty?(node)
node.first_element_child.nil? && node.content.strip.empty?
end

# A one-letter prefix ("h-", "s-") and a counter for repeated headings.
def variants(id)
[id, id.sub(PREFIX, "")].flat_map { |variant| [variant, variant.sub(COUNTER, "")] }
end

# Slugifiers differ in how they treat case, accents, apostrophes and spaces.
def compact(string)
string.unicode_normalize(:nfkd).downcase.gsub(/[^\p{L}\p{N}]/, "")
end
end
end
1 change: 1 addition & 0 deletions lib/nokodown/preprocessor.rb
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ def call
@body.css("pre [class], table[class] td[class]").each { |node| node.remove if node["class"].downcase.match?(LINE_NUMBERS) }
remove_hidden
remove_form_controls
HeadingAnchors.mark(@body)
Dom.assign_keys(@body)
end

Expand Down
7 changes: 4 additions & 3 deletions lib/nokodown/scorer.rb
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,11 @@ def self.class_weight(node)
end

# Conditional cleaning deletes whatever weighs negative, so it ignores CSS
# utility tokens. Candidate scoring keeps Readability's weight, which some
# pages need to pick the right candidate.
# utility tokens and heading anchors ("media" in "h-social-media-tips").
# Candidate scoring keeps Readability's weight, which some pages need to
# pick the right candidate.
def self.cleaning_weight(node)
[node["class"], node["id"]].sum do |value|
[node["class"], Dom.name_id(node)].sum do |value|
tokens = value.to_s.downcase.split.reject { |token| utility_class?(token) }
words_weight(tokens.join(" "))
end
Expand Down
32 changes: 32 additions & 0 deletions spec/nokodown/boilerplate_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,38 @@ def prose(topic, count)
end
end

describe "ids made from a heading's text" do
let(:article) { "<h1>Selling on social media</h1>#{prose('Social selling', 4)}" }

it "keeps headings and sections whose id is their heading's anchor" do
html = <<~HTML
#{article}
<h2 class="wp-block-heading" id="h-what-is-social-shopping">What is social shopping?</h2>#{prose('Social shopping', 1)}
<h3 id="sponsorship-deals-2">Sponsorship deals</h3>#{prose('Sponsorship', 1)}
<section id="cookies"><h2>Cookies<a href="#cookies">¶</a></h2>#{prose('Reading cookies', 1)}</section>
HTML

text = remaining(html)
text.should include("What is social shopping?")
text.should include("Sponsorship deals")
text.should include("Reading cookies is explained")
end

it "still removes junk labels, ids that are not anchors and furniture classes" do
html = <<~HTML
#{article}
<h2 id="related-posts">Related posts</h2>
<section id="share-this-article"><h3>Share this article</h3><a href="/share">Copy link</a></section>
<section id="related-articles"><h2>Related articles<a href="#related-articles">¶</a></h2><a href="/older">Older story</a></section>
<h3 id="social-links">Follow our journey</h3>
<h2 class="social-title" id="h-social-media-tips">Social media tips</h2>
HTML

text = remaining(html)
%w[Related Share Older Follow tips].each { |word| text.should_not include(word) }
end
end

describe "repeated modules" do
def cards(classes, count)
Array.new(count) do |index|
Expand Down
44 changes: 44 additions & 0 deletions spec/nokodown/heading_anchors_spec.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
# frozen_string_literal: true

RSpec.describe Nokodown::HeadingAnchors do
def anchor?(html)
body = Nokolexbor::HTML("<body>#{html}</body>").at_css("body")
described_class.anchor?(body.at_css("[id]"))
end

it "recognizes ids made from a heading's text, with a one-letter prefix or a counter" do
anchor?('<h2 id="h-what-s-next-for-social-shopping">What’s Next for Social Shopping?</h2>').should == true
anchor?('<h3 id="travel-related-coverage-2">Travel-Related Coverage</h3>').should == true
anchor?('<h3 id="Login_shell">Login shell</h3>').should == true
anchor?('<h2 id="H-Social-Tips">Social Tips</h2>').should == true
anchor?('<h2 id="que-es-el-marketing-social">Qué es el marketing social</h2>').should == true
anchor?('<h2><span class="mw-headline" id="Related_to_Portage">Related to Portage</span></h2>').should == true
end

it "recognizes the id of the section a heading opens" do
anchor?('<section id="cookies"><h3>Cookies<a href="#cookies">¶</a></h3><p>Read them from the request.</p></section>').should == true
anchor?('<section id="s-sharing-models"><span id="sharing-models"></span><h4>Sharing models</h4><p>Import them.</p></section>').should == true
end

it "does not take other ids for anchors" do
anchor?('<h3 id="social-links">Follow our journey</h3>').should == false
anchor?('<div id="social-feed"><p>Latest posts.</p><h2>Social feed</h2></div>').should == false
anchor?('<a id="social-sharing">Social sharing</a>').should == false
end

it "does not take the id of a junk label for an anchor, since it names the widget under it" do
anchor?('<h2 id="related-posts">Related posts</h2>').should == false
anchor?('<h3 id="share-this-article">Share this article</h3>').should == false
anchor?('<h2 id="feedback">Feedback</h2>').should == false
anchor?('<h2 id="table-of-contents">Table of contents</h2>').should == false
anchor?('<h2 id="customer-reviews">Customer Reviews</h2>').should == false
anchor?('<section id="ratings-and-reviews"><h2>Ratings and Reviews</h2><p>Sturdy and light.</p></section>').should == false
end

it "reads a junk label without the permalink that anchor markup adds to it" do
anchor?('<section id="comments"><h2>Comments<a href="#comments">¶</a></h2><p>Jane wrote: thanks.</p></section>').should == false
anchor?('<h2 id="related-articles">Related articles <a class="anchor" href="#related-articles">#</a></h2>').should == false
anchor?('<h3 id="share-this-article"><a href="#share-this-article">#</a> Share this article</h3>').should == false
anchor?(%(<h2 class="anchor" id="related-posts">Related posts<a class="hash-link" href="#related-posts">\u200B</a></h2>)).should == false
end
end
9 changes: 9 additions & 0 deletions spec/nokodown/product_reviews_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,15 @@ def markdown(head, main)
result.should_not include("summer of surveys")
end

it "removes a reviews section whose id is made from its heading" do
anchored = '<section id="ratings-and-reviews"><h2>Ratings and Reviews</h2><p>It survived a whole summer of surveys ' \
"in my back pocket, and the paper takes pencil and fountain pen alike.</p></section>"
result = markdown(product_type, "#{description}#{details}#{anchored}")

result.should include("dot-grid paper")
result.should_not include("summer of surveys")
end

it "removes schema.org reviews even though each has a description of its own" do
review = '<div itemprop="review" itemscope itemtype="https://schema.org/Review"><p itemprop="description">' \
"Bought three of these for my survey team and every one of them made it through a wet season of work.</p></div>"
Expand Down
13 changes: 13 additions & 0 deletions spec/nokodown_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -314,6 +314,19 @@ def story(extra)
end
end

it "keeps a heading whose anchor holds a negative class word" do
markdown = story('<h2 id="h-media-coverage-of-tidal-lagoons">Media coverage of tidal lagoons</h2><p>Newspapers mostly covered the cost.</p>')

markdown.should include("## Media coverage of tidal lagoons")
end

it "still drops comments whose heading carries a permalink" do
markdown = story('<section id="comments"><h2>Comments<a href="#comments">¶</a></h2><p>Jane wrote: the turbine part was the clearest.</p></section>')

markdown.should_not include("Comments")
markdown.should_not include("Jane wrote")
end

it "still drops copies shown only at some screen sizes" do
markdown = story('<div class="hidden sm:block"><p>Wide screen copy of the intro.</p></div><div class="md:hidden"><p>Narrow screen copy of the intro.</p></div>')

Expand Down