included.
+ def flatten_deep_elements
+ @body.xpath(".#{'/*' * MAX_DEPTH}[*]").each do |node|
+ text = @body.document.create_text_node(flat_text(node))
+ node.children.remove
+ text_wrappers(node).reverse_each { |name| text = wrap(name, text) }
+ node.add_child(text)
+ end
+ end
+
+ # Text of every descendant, without recursion. Block boundaries and
+ # become line breaks, so words stay apart; keeps them as lines.
+ def flat_text(node)
+ text = +""
+ pending = node.children.to_a.reverse
+ until pending.empty?
+ current = pending.pop
+ if current.is_a?(String)
+ text << current
+ elsif current.text? || current.cdata?
+ text << current.content
+ elsif current.name == "br"
+ text << "\n"
+ elsif current.element?
+ boundary = Dom::BLOCK.include?(current.name)
+ text << "\n" if boundary
+ pending << "\n" if boundary
+ pending.concat(current.children.to_a.reverse)
+ end
+ end
+ text
+ end
+
+ # A in a
is read for its s and s, like the .
+ def text_wrappers(node)
+ return TEXT_WRAPPERS[node.name] if TEXT_WRAPPERS.key?(node.name)
+ return [] unless node.name == "div"
+
+ parent = node.parent
+ parent = parent.parent while parent&.name == "div"
+ parent&.name == "dl" ? %w[dd] : []
+ end
+
+ def wrap(name, child)
+ element = @body.document.create_element(name)
+ element.add_child(child)
+ element
+ end
+
# Accordion triggers usually carry the question text of an FAQ.
def disclosure_button?(button)
parent = button.parent
diff --git a/lib/nokodown/pruner.rb b/lib/nokodown/pruner.rb
index a79ce86..a492178 100644
--- a/lib/nokodown/pruner.rb
+++ b/lib/nokodown/pruner.rb
@@ -130,7 +130,9 @@ def remove_article_furniture(root, content_length)
root.css("aside").each do |node|
remove(node) if Dom.text(node).length < 200 || ContainerCheck.new(node, content_length, Set.new).remove?
end
- root.css("[class], [id]").each do |node|
+ Dom.elements(root).each do |node|
+ next unless node.key?("class") || node.key?("id")
+
remove(node) if Dom.signature(node).match?(SHARE) && Dom.text(node).length < 500
end
end
@@ -166,7 +168,7 @@ def remove_secondary(root, total)
root.element_children.each do |node|
next if Dom.thread?(node)
- if Dom.signature(node).match?(Patterns::SECONDARY) && Dom.text(node).length <= total * 0.5
+ if Patterns.secondary?(Dom.signature(node)) && Dom.text(node).length <= total * 0.5
remove(node)
else
remove_secondary(node, total)
@@ -177,7 +179,7 @@ def remove_secondary(root, total)
# The author line is returned as metadata, like Readability does. Many of
# them mean a thread or a list of posts, where the authors are content.
def remove_bylines(root)
- bylines = root.css("*").select do |node|
+ bylines = Dom.elements(root).select do |node|
byline = node["rel"] == "author" || node["itemprop"].to_s.include?("author") || Dom.signature(node).match?(BYLINE)
byline && Dom.text(node).length.between?(1, 99)
end
@@ -248,8 +250,8 @@ def remove_calls_to_action(root)
end
def destination(link)
- href = link["href"].to_s.strip
- href unless href.empty? || href.start_with?("#") || href.match?(/\Ajavascript:/i)
+ href = UrlResolver.normalize(link["href"])
+ href unless href.empty? || href.start_with?("#") || UrlResolver.script?(href)
end
def links_of(node)
diff --git a/lib/nokodown/scorer.rb b/lib/nokodown/scorer.rb
index fff3a44..9d7a272 100644
--- a/lib/nokodown/scorer.rb
+++ b/lib/nokodown/scorer.rb
@@ -13,7 +13,8 @@ class Scorer
SECTIONS_SHARE = 0.75
SECTION_LINK_DENSITY = 0.25
TOP_CANDIDATES = 5
- COMMAS = /[,\u060C\uFE50\uFE10\uFE11\u2E41\u2E34\u2E32\uFF0C]/.freeze
+ # The commas Readability counts, for String#count.
+ COMMAS = ",\u060C\uFE50\uFE10\uFE11\u2E41\u2E34\u2E32\uFF0C"
def self.class_weight(node)
[node["class"], node["id"]].sum { |value| words_weight(value.to_s.downcase) }
@@ -23,9 +24,11 @@ def self.class_weight(node)
# utility tokens and heading anchors ("media" in "h-social-media-tips").
# Candidate scoring keeps Readability's weight, which some pages need to
# pick the right candidate.
- def self.cleaning_weight(node)
+ # Words matching ignore count for nothing.
+ def self.cleaning_weight(node, ignore: nil)
[node["class"], Dom.name_id(node)].sum do |value|
tokens = value.to_s.downcase.split.reject { |token| utility_class?(token) }
+ tokens = tokens.map { |token| token.gsub(ignore, "") } if ignore
words_weight(tokens.join(" "))
end
end
@@ -74,8 +77,7 @@ def score(node)
def score_paragraphs
ignored = secondary_keys
- @root.traverse do |node|
- next unless node.element?
+ Dom.elements_bottom_up(@root).each do |node|
next if ignored.include?(Dom.key(node))
if SCORED.include?(node.name) || paragraph?(node)
@@ -90,10 +92,10 @@ def score_paragraphs
# article. A thread holding most of the page (a forum) still counts.
def secondary_keys
limit = @stats.text_chars(@root) * 0.5
- regions = @root.css("*").select do |node|
- Dom.signature(node).match?(Patterns::SECONDARY) && @stats.text_chars(node) < limit
+ regions = Dom.elements(@root).select do |node|
+ Patterns.secondary?(Dom.signature(node)) && @stats.text_chars(node) < limit
end
- regions.flat_map { |region| [region] + region.css("*").to_a }.map { |node| Dom.key(node) }.to_set
+ regions.flat_map { |region| [region] + Dom.elements(region) }.map { |node| Dom.key(node) }.to_set
end
# A div wrapping a single paragraph is that paragraph, as far as scoring goes.
@@ -121,7 +123,8 @@ def phrasing_runs(div)
def add_score(text, parent)
return if text.length < PARAGRAPH_MIN || parent.nil? || !within_root?(parent)
- points = 1 + text.split(COMMAS, -1).size + [text.length / 100, 3].min
+ # Readability counts the pieces between commas; text here is never empty.
+ points = 2 + text.count(COMMAS) + [text.length / 100, 3].min
ancestors_from(parent).each_with_index do |ancestor, level|
key = Dom.key(ancestor)
@nodes[key] ||= ancestor
@@ -150,8 +153,15 @@ def ancestors_from(parent)
list
end
+ # A walk up the parents: Node#ancestors builds a NodeSet of every
+ # ancestor first, which cost more than the rest of the scoring.
def within_root?(node)
- Dom.key(node) == @root_key || node.ancestors.any? { |ancestor| ancestor.element? && Dom.key(ancestor) == @root_key }
+ while node
+ return true if node.element? && Dom.key(node) == @root_key
+
+ node = node.parent
+ end
+ false
end
def initial_score(node)
@@ -248,8 +258,8 @@ def section?(sibling, top, threshold)
def best_score_within(node)
best = -Float::INFINITY
- node.traverse do |descendant|
- value = descendant.element? && score(descendant)
+ Dom.elements_bottom_up(node).each do |descendant|
+ value = score(descendant)
best = value if value && value > best
end
best
diff --git a/lib/nokodown/stats.rb b/lib/nokodown/stats.rb
index 7b8b492..ce53d00 100644
--- a/lib/nokodown/stats.rb
+++ b/lib/nokodown/stats.rb
@@ -41,7 +41,8 @@ def lookup(table, node)
def measure(node, link_weight)
chars = 0.0
link_chars = 0.0
- node.children.each do |child|
+ child = node.child
+ while child
if child.text? || child.cdata?
count = Dom.char_count(child.content)
chars += count
@@ -51,6 +52,7 @@ def measure(node, link_weight)
chars += child_chars
link_chars += child_link_chars
end
+ child = child.next_sibling
end
store(node, chars, link_chars)
end
diff --git a/lib/nokodown/tables.rb b/lib/nokodown/tables.rb
index d0c703a..3eada58 100644
--- a/lib/nokodown/tables.rb
+++ b/lib/nokodown/tables.rb
@@ -26,8 +26,12 @@ def data_table?(table)
# however many rows they have. Hacker News is the canonical example.
def sparse?(table)
cells = table.css("td")
- blank = cells.count { |cell| cell.content.strip.empty? && cell.at_css("img").nil? }
- blank > cells.size * 0.3
+ cells.count { |cell| blank?(cell) } > cells.size * 0.3
+ end
+
+ # No text, image or player.
+ def blank?(cell)
+ cell.content.strip.empty? && cell.at_css("img, video, audio").nil?
end
def size(table)
diff --git a/lib/nokodown/url_resolver.rb b/lib/nokodown/url_resolver.rb
index 7547091..3138a68 100644
--- a/lib/nokodown/url_resolver.rb
+++ b/lib/nokodown/url_resolver.rb
@@ -58,40 +58,73 @@ class UrlResolver
PRIVATE_BUCKET = /\Abucketeer-/i.freeze
WEB_URL = %r{\Ahttps?://}i.freeze
UNSAFE = %r{[^A-Za-z0-9\-._~:/?#@!$&'()*+,;=%]}.freeze
+ # Schemes that run code or open a document built from the URL itself.
+ SCRIPT_SCHEME = /\A(?:javascript|vbscript|data):/i.freeze
+ # A browser ignores spaces and control characters around a URL, and tabs
+ # and line breaks anywhere in it ("java script:" is "javascript:").
+ VISIBLE = /[^\x00-\x20]/.freeze
+
+ # The URL a browser would read from an attribute value. Trimmed by index:
+ # on Ruby 2.7 a regex for trailing controls rescans a run of spaces
+ # inside the URL from each of its characters.
+ def self.normalize(href)
+ href = href.to_s
+ first = href.index(VISIBLE)
+ return "" unless first
+
+ href[first..href.rindex(VISIBLE)].delete("\t\n\r")
+ end
+
+ def self.script?(url)
+ url.match?(SCRIPT_SCHEME)
+ end
def initialize(url, base_href = nil)
@base = parse(url)
- href = base_href.to_s.strip
- @base = join(@base, href) || @base if !href.empty? && !href.start_with?("javascript:")
+ href = UrlResolver.normalize(base_href)
+ base = join(@base, href) unless href.empty? || UrlResolver.script?(href)
+ @base = base if base && !UrlResolver.script?(base.to_s)
+ # A page links to the same places many times, and content selection
+ # and rendering both resolve every link.
+ @links = {}
+ @images = {}
+ @same_page = {}
end
def resolve(href)
- absolute(href) { |uri| clean_link(uri) }
+ @links.fetch(href) { @links[href.dup.freeze] = absolute(href) { |uri| clean_link(uri) }.freeze }
end
def resolve_image(src)
- absolute(src) { |uri| clean_image(uri) }
+ @images.fetch(src) { @images[src.dup.freeze] = absolute(src) { |uri| clean_image(uri) }.freeze }
end
def same_page?(url)
+ @same_page.fetch(url) { @same_page[url.dup.freeze] = page_anchor?(url) }
+ end
+
+ private
+
+ def page_anchor?(url)
return false unless @base
uri = parse(url)
return false unless uri
@page ||= strip_fragment(clean_link(@base))
- uri.fragment && strip_fragment(uri) == @page
+ !uri.fragment.nil? && strip_fragment(uri) == @page
end
- private
-
+ # Nothing that runs a script, before or after resolving: a could
+ # lend its scheme to a relative path.
def absolute(href)
- href = href.to_s.strip.delete("\t\n\r")
- return if href.empty?
- return href if href.match?(/\A(?:mailto|tel|data):/i)
+ href = UrlResolver.normalize(href)
+ return if href.empty? || UrlResolver.script?(href)
+ return href if href.match?(/\A(?:mailto|tel):/i)
uri = @base ? join(@base, href) : parse(href)
- uri ? yield(uri).to_s : href
+ url = uri ? yield(uri).to_s : href
+ url unless UrlResolver.script?(url)
end
def parse(url)
diff --git a/spec/nokodown/boilerplate_spec.rb b/spec/nokodown/boilerplate_spec.rb
index dcad884..be2fb6d 100644
--- a/spec/nokodown/boilerplate_spec.rb
+++ b/spec/nokodown/boilerplate_spec.rb
@@ -68,6 +68,29 @@ def prose(topic, count)
end
end
+ describe "furniture class names" do
+ it "finds a furniture name anywhere in a class list" do
+ html = <<~HTML
+ Heat pumps move heat instead of making it.
+
+ We use cookies
+ HTML
+
+ remaining(html).should == "Heat pumps move heat instead of making it."
+ end
+
+ # Boilerplate matches and caches each class name on its own, which only
+ # gives the same answer as matching the whole list while no pattern can
+ # match across the space between two names.
+ it "has no pattern that matches across class names" do
+ pattern = Nokodown::Patterns::PAGE_FURNITURE
+ pattern.source.should_not match(/\\[sSWDhH ]|\[\^|(?Choosing a heat pump#{prose('Sizing', 6)}" }
diff --git a/spec/nokodown/dom_spec.rb b/spec/nokodown/dom_spec.rb
new file mode 100644
index 0000000..c227fa8
--- /dev/null
+++ b/spec/nokodown/dom_spec.rb
@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+RSpec.describe Nokodown::Dom do
+ def element(attributes)
+ Nokolexbor::HTML("
").at_css("div")
+ end
+
+ describe ".paragraph_keys" do
+ def keyed(html)
+ body = Nokolexbor::HTML("#{html}").at_css("body")
+ described_class.assign_keys(body)
+ body
+ end
+
+ def paragraphs(body, root = body)
+ keys = described_class.paragraph_keys(root)
+ body.css("[id]").select { |node| keys.include?(described_class.key(node)) }.map { |node| node["id"] }
+ end
+
+ it "finds paragraphs and the divs without block children Readability rewrites into them" do
+ body = keyed(<<~HTML)
+
+ HTML
+
+ paragraphs(body).should == %w[text para]
+ paragraphs(body, body.at_css("#text")).should == %w[text]
+ end
+
+ # Lexbor copies a link that is cut by a block into several elements that
+ # share one attribute list, so the copies share a key too. The copy
+ # visited last in a bottom-up pass decides, as it always has.
+ it "lets the last copy of a misnested link decide whether it holds a block" do
+ paragraphs(keyed('')).should ==
+ %w[wrapper]
+ paragraphs(keyed('')).should == []
+ end
+ end
+
+ describe ".elements" do
+ it "lists the elements under a root in document order, as css(\"*\") does" do
+ html = <<~HTML
+ Title text
+ HTML
+ body = Nokolexbor::HTML(html).at_css("body")
+
+ described_class.elements(body).map(&:name).should == body.css("*").map(&:name)
+ described_class.elements(body.at_css("section")).map(&:name).should == %w[p b i ul li li a]
+ described_class.elements(body.at_css("b")).should == []
+ end
+
+ # Copies of a formatting element a block cuts share one attribute list
+ # in Lexbor; the walk must still visit each copy.
+ it "visits every copy of a misnested formatting element" do
+ body = Nokolexbor::HTML('Offer C
').at_css("body")
+
+ described_class.assign_keys(body)
+ described_class.shared_keys?(body).should == true
+
+ described_class.elements(body).map(&:name).should == body.css("*").map(&:name)
+ end
+
+ it "keys every element in document order" do
+ body = Nokolexbor::HTML("z
").at_css("body")
+ described_class.assign_keys(body)
+
+ body.css("*").map { |node| described_class.key(node) }.should == %w[1 2 3 4 5]
+ end
+ end
+
+ describe ".elements_bottom_up" do
+ def traversed(root)
+ found = []
+ root.traverse { |node| found << node if node.element? }
+ found
+ end
+
+ it "lists root and its elements children first, as traverse yields them" do
+ body = Nokolexbor::HTML("Title textEnd
").at_css("body")
+
+ described_class.elements_bottom_up(body).map(&:name).should == %w[h1 b p li ul section main p body]
+ described_class.elements_bottom_up(body).map(&:name).should == traversed(body).map(&:name)
+ described_class.elements_bottom_up(body.at_css("b")).map(&:name).should == %w[b]
+ end
+
+ it "visits every copy of a misnested formatting element" do
+ body = Nokolexbor::HTML('Offer C
').at_css("body")
+ described_class.assign_keys(body)
+ described_class.shared_keys?(body).should == true
+
+ described_class.elements_bottom_up(body).map { |node| [node.name, described_class.key(node)] }
+ .should == traversed(body).map { |node| [node.name, described_class.key(node)] }
+ end
+ end
+
+ describe ".ancestors_named" do
+ it "lists the ancestors with the given names, nearest first" do
+ doc = Nokolexbor::HTML("")
+
+ described_class.ancestors_named(doc.at_css("b"), "td", "p", "table").map(&:name).should == %w[p td table]
+ described_class.ancestors_named(doc.at_css("b"), "figure").should == []
+ end
+
+ it "climbs out of a copy or a fragment without failing" do
+ doc = Nokolexbor::HTML("Cell
")
+ copy = doc.at_css("p").dup
+
+ described_class.ancestors_named(copy.at_css("b"), "p", "figure").map(&:name).should == %w[p]
+ described_class.ancestors_named(doc.fragment("x ").at_css("b"), "i").map(&:name).should == %w[i]
+ end
+ end
+
+ describe ".signature" do
+ it "lowercases class and id and splits camelCase" do
+ described_class.signature(element('class="adContainer siteNav2Bar" id="mainHeader"')).should ==
+ "ad-container site-nav2-bar main-header"
+ end
+
+ it "drops generated ids and CSS module hashes" do
+ node = element('class="w-node-b5343ad4-1c2e-ad12-9f0b-3c4d5e6f7a8b mdx-table_root__adKqa"')
+ described_class.signature(node).should == "w-node- mdx-table_root "
+ end
+
+ it "is empty without class and id" do
+ described_class.signature(element('title="x"')).should == ""
+ end
+
+ it "gives the same signature from the cache an extraction keeps" do
+ nodes = %w[adContainer adContainer cookieBanner].map { |name| element("class=\"#{name}\" id=\"x#{name}\"") }
+ uncached = nodes.map { |node| described_class.signature(node) }
+
+ cached = described_class.caching_signatures do
+ Array.new(2) { nodes.map { |node| described_class.signature(node) } }
+ end
+
+ cached.should == [uncached, uncached]
+ Thread.current[described_class::SIGNATURES].should be_nil
+ end
+ end
+end
diff --git a/spec/nokodown/markdown_spec.rb b/spec/nokodown/markdown_spec.rb
index d5cd5a2..1bde7ea 100644
--- a/spec/nokodown/markdown_spec.rb
+++ b/spec/nokodown/markdown_spec.rb
@@ -6,6 +6,11 @@
convert(" one\n two three
").should == "one two three"
end
+ it "collapses Unicode spaces and drops invisible characters, in ASCII and non-ASCII text" do
+ convert("one\ttwo\r\nthree
caf\u00E9\u00A0au\u3000lait, co\u00ADoperate, zero\u200Bwidth
").should ==
+ "one two three\n\ncaf\u00E9 au lait, cooperate, zerowidth"
+ end
+
it "splits paragraphs on a double " do
convert("first line second line next paragraph
").should == "first line\nsecond line\n\nnext paragraph"
end
@@ -14,6 +19,16 @@
convert("snake_case, 5 * 3, *starred*, `tick`, bold
").should == "snake_case, 5 * 3, \\*starred\\*, \\`tick\\`, **bold**"
end
+ it "escapes brackets, angle brackets, backslashes and lone underscores" do
+ convert("[a](b) <b> C:\\* _x_ plain text
").should == "[a\\](b) \\ C:\\\\\\* \\_x\\_ plain text"
+ end
+
+ it "doubles a backslash at the end of a text, so it escapes nothing after it" do
+ convert(' /!\\ Restart /!\\ hosted only.\\ next
').should == "**/!\\ Restart /!\\\\** hosted only.\\\\\nnext"
+ convert('a\\[Run](javascript:alert(1))[ [Run](javascript:alert(1))[
').should ==
+ "a\\\\`[Run](javascript:alert(1))[` [`[Run](javascript:alert(1))[`](https://example.com/x)"
+ end
+
it "escapes text that would start a heading or a quote" do
convert("# not a heading
> not a quote
").should == "\\# not a heading\n\n\\> not a quote"
end
@@ -106,6 +121,21 @@
"[Guide](https://example.com/guide) and [about](https://example.com/about)"
end
+ it "takes the target of a link without href from its data attributes" do
+ convert('Visit Article .
').should == "Visit [Article](https://example.com/article)."
+ convert('Fallback
').should == "[Fallback](https://example.com/fallback)"
+ convert('Real
').should == "[Real](https://example.com/real)"
+ end
+
+ it "takes the target from its data attributes when href is blank" do
+ convert('Visit Article .
').should == "Visit [Article](https://example.com/article)."
+ convert('Visit News .
').should == "Visit [News](https://example.com/news)."
+ end
+
+ it "keeps only the text of data attribute targets that are templates or scripts" do
+ convert('Mail Share
').should == "Mail Share"
+ end
+
it "strips click-tracking parameters" do
convert('Item
').should == "[Item](https://shop.example/p?id=7)"
convert('Item
').should == "[Item](https://shop.example/p?id=7)"
@@ -173,14 +203,161 @@
convert('Top Open Call
').should == "Top Open Call"
end
+ it "keeps only the text of script links that tabs, line breaks or control characters disguise" do
+ convert('Tab Line ' \
+ 'Control Basic
').should ==
+ "Tab Line Control Basic"
+ convert('Clip
').should == "Clip"
+ end
+
+ it "does not take a script scheme from " do
+ html = 'Guide
'
+ Nokodown.to_markdown(html, url: "https://example.com/", main_content: false, title: false).should ==
+ "[Guide](https://example.com/guide) "
+ end
+
+ it "writes a URL so that Markdown's escapes and entity references cannot turn it into a script" do
+ html = 'Slash Entity ' \
+ 'Code Query
'
+ Nokodown.to_markdown(html, main_content: false, title: false).should ==
+ "[Slash](javascript%5C:alert(1)) [Entity](javascript%26colon;alert(1)) [Code](%26#x6a;avascript:alert(1)) [Query](/search?q=a&b=c)"
+ end
+
+ it "encodes the entities of a long non-ASCII URL in linear time" do
+ url = "/%#{'é&' * 64_000}"
+ started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+ Nokodown.to_markdown(%(Go ), main_content: false, title: false).should == "[Go](/%#{'é&' * 64_000})"
+ (Process.clock_gettime(Process::CLOCK_MONOTONIC) - started).should < 1
+ end
+
+ it "writes a URL so that Markdown reads it as one destination" do
+ html = 'Mail Pair ' \
+ 'Tab Wiki
' \
+ ' '
+ convert(html).should == "[Mail](mailto:x%29%5BRun%5D%28javascript:alert%281%29%29%28) [Pair](mailto:x(a)%5BRun%5D(javascript:alert(1))) " \
+ "[Tab](mailto:x%0B%29%5BRun%5D%28javascript:alert%281%29%29) [Wiki](https://example.com/wiki/Coffee_(disambiguation))\n\n" \
+ "[Audio](https://example.com/a.mp3%29%5BRun%5D%28javascript:alert%281%29%29%28)"
+ convert("Deep ").should == "[Deep](https://example.com/x#{'%28' * 33}y#{'%29' * 33})"
+ end
+
+ it "writes a URL so that it holds no link or code of its own, should text before it keep the link from forming" do
+ html = 'a\\Go
' \
+ 'Go
' \
+ 'IPv6 Filter
'
+ convert(html).should == "a\\\\[Go](mailto:%5BRun%5D(javascript:alert(1)))\n\n$a\\`b$ [Go](mailto:x%60%5BRun%5D(javascript:alert(1)))\n\n" \
+ "[IPv6](http://[2001:db8::1]/a%5Bb%5D) [Filter](https://example.com/search?f%5B0%5D=a)"
+ end
+
+ it "keeps only the brackets of an IPv6 host as they are, and encodes its user info" do
+ html = 'Tag Space ' \
+ 'Path
'
+ convert(html).should == "[Tag](http://%3Cimg%20src=x%3E@[::1]/) [Space](http://a%20b@[::1]:8080/x%20y) " \
+ "[Path](http://[2001:db8::1]/a%5Bb%5D)"
+ end
+
it "drops permalink symbols and read-more links" do
convert('Setup¶ Intro. Read more »
').should == "## Setup\n\nIntro."
end
+ it "ends link text at its own bracket, whatever backslashes and brackets it holds" do
+ html = 'Go\\ a\\] ]x[ ' \
+ '[1] note
'
+ convert(html).should == "[Go\\\\](mailto:%5BRun%5D(javascript:alert(1))) [a\\\\\\]](https://example.com/x) " \
+ "[\\]x\\[](https://example.com/x) [[1] note](https://example.com/x)\n\n"
+ end
+
+ it "reads code in link text as Markdown does, keeping its brackets and backslashes as they are" do
+ html = '][ Go [] ' \
+ '[\\]] a `b`c
'
+ convert(html).should == "[`][`](https://example.com/x) [Go `[`\\]](mailto:%5BRun%5D(javascript:alert(1))) " \
+ "[`[\\]`\\]](https://example.com/z) [`a`\\```b`c``](https://example.com/w)"
+ end
+
+ it "writes no link or image inside link text, whatever TeX it holds" do
+ html = '[ ' \
+ ' ' \
+ '
'
+ convert(html).should == "[`[`$\\[Run\\](javascript:alert(1))$](https://example.com/x) " \
+ "[$\\[\\[Run\\](javascript:alert(1))$](https://example.com/y) " \
+ "[$!\\[i\\](javascript:alert(1))$](https://example.com/z)"
+ end
+
+ it "reads no backtick run longer than Markdown's fence limit as code in link text" do
+ ticks = "`" * 80
+ convert(%(#{ticks}[Run](javascript:alert(1))[
)).should ==
+ "[#{'\\`' * 80}\\[Run\\](javascript:alert(1))\\[](https://example.com/x)"
+ convert(%(#{'`' * 79}[a]
)).should == "[#{ticks} #{'`' * 79}[a] #{ticks}](https://example.com/x)"
+ end
+
+ it "writes no autolink or HTML tag inside link text, whatever TeX it holds" do
+ html = '``x`` ' \
+ '
'
+ convert(html).should == "[``` ``x`` ```$\\\\`\\`$](https://example.com/x) " \
+ "[$\\ $](https://example.com/y) [$a < 1$](https://example.com/z)"
+ end
+
+ it "writes no email autolink in text, link text or a player's title, whatever its address starts with" do
+ html = 'Mail <0@host.com> or <.a@host.com>, not a < 1
' \
+ '``x`` ' \
+ 'a<0a`bc@host.com>
'
+ convert(html).should == "Mail \\<0@host.com> or \\<.a@host.com>, not a < 1\n\n" \
+ "[``` ``x`` ```$\\<0@host.com>\\`\\`$](https://example.com/x) [`a`\\<0``a`b``c@host.com>](https://example.com/y)\n\n" \
+ "[\\<0@host.com>](https://example.com/a.mp3)"
+ end
+
+ it "keeps a space between a code span and a backtick after it, so their runs do not merge" do
+ convert("a``x Ctrl C C:\\x x`y
").should ==
+ "`a` ``` ``x ``` `Ctrl` `C` `C:\\` `x` x\\``y`"
+ convert("x`q` y C:\\x x`y
").should ==
+ "x\\`q\\``y` `C:\\` `x` x\\``y`"
+ convert('` x ` y C:\\z
').should ==
+ "\\``x` \\``y` `C:\\` `z`"
+ convert('a``x [Run](javascript:alert(1))[
').should ==
+ "`a` ``` ``x ``` [`[Run](javascript:alert(1))[`](https://example.com/x)"
+ end
+
+ it "writes code that ends with a backslash unpadded, so a link or heading of 300 characters keeps its form" do
+ code = "a" * 297 + "\\"
+ convert(%(#{code}
)).should == "[`#{code}`](https://example.com/x)"
+ convert(%(#{code}x
)).should == "## `#{code}`\n\nx"
+ text = "a" * 295
+ convert(%(#{text}` x
)).should == "[#{text}\\``x`](https://example.com/x)"
+ convert(%(#{text}` x x
)).should == "## #{text}\\``x`\n\nx"
+ end
+
+ it "escapes every backtick of a run that would open a code block, and keeps one that opens a code span" do
+ convert("```x y ``z
").should == "\\`\\`\\`x\ny \\`\\`z"
+ convert('``x ``[Run](javascript:alert(1))[
').should ==
+ "``` ``x ``` [``` ``[Run](javascript:alert(1))[ ```](https://example.com/x)"
+ end
+
+ it "escapes the link, image, autolink or tag that pieces of text side by side would write" do
+ html = 'Wow!x [1](note) <javascript:alert(1)> <0@h.co> ' \
+ '<https://b.org/doc >
'
+ convert(html).should == "Wow\\ [1\\](note) \\ \\<0`@h.co>` "
+ convert(%q{<a title=' '>[Run](javascript:alert(1))[
}).should ==
+ "\\[Run](javascript:alert(1))[`](https://example.com/x)"
+ convert('<0a`bc@host.com> <user@example.com> <user@example.com>
').should ==
+ "\\<0``a`b``c@host.com> `<`user@example.com> [`<`user@example.com>](https://example.com/x)"
+ end
+
+ it "writes code with a backtick run too long to fence as text" do
+ ticks = "`" * 80
+ convert(%(#{ticks}[Run](javascript:alert(1)) #{'`' * 79}a
)).should ==
+ "#{'\\`' * 80}[Run\\](javascript:alert(1)) #{ticks} #{'`' * 79}a #{ticks}"
+ end
+
it "writes a link whose text is its URL as a bare URL" do
convert('example.org
').should == "https://example.org/"
end
+ it "writes a link whose URL holds Markdown as a link, even when its text is the URL" do
+ html = '[Run](javascript:alert(1)) ' \
+ 'https://example.org/a`b
'
+ convert(html).should == "[\\[Run\\](javascript:alert(1))](mailto:%5BRun%5D(javascript:alert(1))) " \
+ "[https://example.org/a\\`b](https://example.org/a%60b)"
+ end
+
it "puts images out of link text" do
convert(' Trail boot
').should ==
" [Trail boot](https://example.com/p/1)"
@@ -201,6 +378,105 @@
convert(' ').should == ""
end
+ it "uses lazy-loading attributes over a placeholder or thumbnail src without a lazy class" do
+ convert('Visible
').should == "Visible\n\n"
+ convert('Visible
').should == "Visible\n\n"
+ convert(' ').should == ""
+ end
+
+ it "takes the real image from lazy-loading attributes in priority order, over a placeholder or thumbnail src" do
+ names = %w[data-a-hires data-src-hq data-client-src-hq data-original data-full-src data-original-src data-src data-lazy-src
+ data-lazy data-lazysrc data-bsrc data-iurl data-thumb data-image data-img data-a-image-source data-fallback-src]
+ names.each do |name|
+ convert(%( )).should == ""
+ end
+ convert(' ').should ==
+ ""
+ convert(' ').should ==
+ ""
+ convert(' ').should ==
+ " "
+ end
+
+ it "prefers a high-resolution attribute to a thumbnail or a dynamic image map" do
+ convert(' ').should ==
+ ""
+ %w[data-original data-a-hires].each do |name|
+ convert(%( )).should ==
+ ""
+ end
+ convert(%( )).should ==
+ ""
+ end
+
+ it "takes the largest of Amazon's dynamic image sizes" do
+ sizes = '{"https://m.media-amazon.com/I/a._SX300_.jpg":[300,300],"https://m.media-amazon.com/I/a._SX679_.jpg":[679,679]}'
+ convert(%( )).should ==
+ ""
+ convert(%( )).should ==
+ " "
+ end
+
+ it "keeps src when the lazy-loading attributes hold no image location" do
+ html = ' ' \
+ '
'
+ convert(html).should == "  "
+ end
+
+ it "keeps src when a lazy srcset or a dynamic image map holds no image location" do
+ %w[false 123].each do |value|
+ convert(%( )).should == ""
+ convert(%( )).should == ""
+ convert(%( )).should ==
+ ""
+ end
+ end
+
+ it "takes no spacer file, word or fragment from the lazy-loading attributes" do
+ convert(' ').should == ""
+ convert(' ').should == ""
+ convert(' ').should ==
+ ""
+ convert(' ').should == ""
+ end
+
+ it "takes any file name or path from the lazy-loading attributes of a marked image" do
+ html = ' ' \
+ ' ' \
+ '
'
+ convert(html).should == "  " \
+ ""
+ end
+
+ it "takes a query or an endpoint from a marked image's lazy-loading attributes" do
+ html = ' ' \
+ '
'
+ convert(html).should == "  " \
+ ""
+ end
+
+ it "takes a bare relative URL from a marked image without src, but no word a template wrote for a missing value" do
+ html = ' ' \
+ '
'
+ convert(html).should == "  " \
+ ""
+ end
+
+ it "takes the first bare relative URL from a marked image whose src is empty, before a later path" do
+ html = ' ' \
+ '
'
+ convert(html).should == " "
+ end
+
+ it "reads a marked image's lazy-loading file name with a query, in linear time" do
+ convert(' ').should ==
+ ""
+
+ started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+ convert(%( )).should start_with("
+ (Process.clock_gettime(Process::CLOCK_MONOTONIC) - started).should < 0.5
+ end
+
it "uses the copy of a lazy image once, on either side of the placeholder" do
html = ' ' \
' ' \
@@ -220,14 +496,16 @@
convert(html).should == "Text"
end
- it "uses the alt text of emoji images" do
+ it "uses the alt text of emoji images, escaped like any text" do
convert('
Done
').should == "Done 🎉"
+ convert('[Run](javascript:alert(1))[
').should ==
+ "\\`[`[Run](javascript:alert(1))[`](https://example.com/x)"
end
it "drops repeated responsive variants of one image" do
html = 'Credit ' \
'Credit '
- convert(html).should == "\n\nCredit"
+ convert_content(html).should == "\n\nCredit"
end
it "drops image CDN size, quality and version parameters" do
@@ -302,7 +580,7 @@
html = 'Trusted by ' \
'' \
'Rated by 2,000 teams
'
- convert(html).should == "## Trusted by\n\n- \n- \n\n" \
+ convert_content(html).should == "## Trusted by\n\n- \n- \n\n" \
"Rated  by 2,000 teams"
end
@@ -318,20 +596,20 @@
html = %w[Member Moderator Member].each_with_index.map do |role, i|
"Post #{i + 1}
"
end.join
- convert(html).should == "\n\nPost 1\n\n" \
+ convert_content(html).should == "\n\nPost 1\n\n" \
"\n\nPost 2\n\n\n\nPost 3"
end
it "keeps nested list markers when it drops a repeated image" do
html = ' Forum
First post
' \
'Reply
'
- convert(html).should == " Forum\n\n1. First post\n\n 1. - Reply"
+ convert_content(html).should == " Forum\n\n1. First post\n\n 1. - Reply"
end
it "does not remove repeated images from code blocks, nested ones included" do
code = "\n\n\nend "
html = %(
#{code}Use this:#{code} )
- convert(html).should == "\n\n```\n\n\n\nend\n```\n\n" \
+ convert_content(html).should == "\n\n```\n\n\n\nend\n```\n\n" \
"- Use this:\n\n ```\n \n\n\n end\n ```"
end
@@ -350,7 +628,7 @@ def extract(html)
extract(html).should == "# Hiking boots\n\n" \
"- ### [Trail boot](https://example.com/p/1)\n\n Waterproof leather for long days on rocky ground.\n\n" \
"- ### [Ridge runner](https://example.com/p/2)\n\n A light mesh boot for fast summer hikes.\n\n" \
- "- "
+ "- [](https://example.com/lookbook.jpg)"
end
it "moves a card's list marker past a nested list of thumbnails" do
@@ -373,10 +651,20 @@ def extract(html)
'' \
''
extract(html).should == "# Case studies\n\nHow our customers ship faster with our software.\n\n" \
- "\n\nAcme cut picking time in half.\n\n" \
- "\n\n[$40](https://example.com/c/2)\n\n" \
- "\n\n[Oct 1, 2026](https://example.com/c/4)\n\n" \
- ""
+ "[](https://example.com/c/1)\n\nAcme cut picking time in half.\n\n" \
+ "[](https://example.com/c/2)\n\n[$40](https://example.com/c/2)\n\n" \
+ "[](https://example.com/c/4)\n\n[Oct 1, 2026](https://example.com/c/4)\n\n" \
+ "[](https://example.com/c/3)"
+ end
+
+ it "keeps the destination of a link made only of an image, in a heading too" do
+ extract('Portrait
')
+ .should == "Portrait\n\n[](https://example.com/full)"
+ extract('Instructions follow
')
+ .should == "[](https://example.com/manual)\n\nInstructions follow"
+ Nokodown.to_markdown('Portrait
',
+ url: "https://example.com/", main_content: false, title: false)
+ .should == "Portrait\n\n[](https://example.com/full)"
end
it "keeps article images unless a short alt repeats a text link to the same page" do
@@ -406,7 +694,8 @@ def extract(html)
markdown.should include("")
markdown.should include("Start here: [Tides explained](https://example.com/tides)")
- markdown.should include("Or:  [Severn guide](https://example.com/severn)")
+ markdown.should include("Or: [](https://example.com/severn) " \
+ "[Severn guide](https://example.com/severn)")
end
it "drops images marked decorative, but not the ticks of a comparison table" do
@@ -606,6 +895,21 @@ def code_block(code)
"| Key | Value |\n| --- | --- |\n| a\\|b | 1 |"
end
+ it "keeps a backslash before a pipe in a table cell" do
+ convert('').should ==
+ "| Pattern |\n| --- |\n| `a\\\\|b` |"
+ convert('').should ==
+ "| Key | Value |\n| --- | --- |\n| a\\\\\\|b | 1 |"
+ end
+
+ it "escapes the pipes of a table nested in a cell once, for the outer table" do
+ inner = ''
+ convert("").should ==
+ "| O |\n| --- |\n| \\| I \\| \\| --- \\| \\| `a\\|b` x\\\\\\|y \\| |"
+ convert("").should ==
+ "| O |\n| --- |\n| \\| M \\| \\| --- \\| \\| \\| I \\| \\| --- \\| \\| `a\\|b` x\\\\\\|y \\| \\| |"
+ end
+
it "carries rowspans down and leaves colspans blank" do
html = "Group Item Qty " \
'Fruit Apple 3 Pear 5 ' \
@@ -618,6 +922,29 @@ def code_block(code)
"Left column\n\nRight column"
end
+ it "keeps players in layout table cells" do
+ convert('').should == "[Video](https://example.com/clip.mp4)"
+ html = ''
+ convert(html).should == "Intro\n\n[Audio](https://example.com/a.mp3)\n\nOutro\n\n[Video](https://example.com/v.mp4)"
+ end
+
+ it "does not count a cell holding a player as blank" do
+ row = 'Talk 12 min '
+ convert("").should start_with("| Talk | [Video](https://example.com/talk.mp4) | 12 min |")
+ end
+
+ it "renders a single column under a header cell or a caption as a table" do
+ convert("").should == "| Col |\n| --- |\n| a\\|b |"
+ convert("").should ==
+ "Sizes\n\n| S |\n| --- |\n| M |"
+ Nokodown.to_markdown("").should == "| Col |\n| --- |\n| a\\|b |"
+ end
+
+ it "unwraps a single column with neither header cell nor caption" do
+ convert("").should == "One\n\nTwo"
+ end
+
it "keeps the caption" do
convert("").should ==
"Results\n\n| A | B |\n| --- | --- |\n| 1 | 2 |"
@@ -630,7 +957,7 @@ def code_block(code)
end
it "keeps one rule between blocks" do
- convert("a
b
").should == "a\n\n---\n\nb"
+ convert_content("a
b
").should == "a\n\n---\n\nb"
end
it "renders details with a bold summary" do
@@ -642,16 +969,115 @@ def code_block(code)
"[Demo video](https://www.youtube.com/watch?v=dQw4w9WgXcQ)"
end
+ it "links an embed without a title as a video, and only from a video host" do
+ convert('Watch
').should ==
+ "Watch\n\n[Video](https://player.vimeo.com/video/123)"
+ convert('Watch
').should == "Watch"
+ convert('Watch
').should == "Watch"
+ convert('Watch
').should ==
+ "Watch\n\n[Video](https://player.vimeo.com/video/1?q=%5BRun%5D(javascript:alert(1)))"
+ end
+
+ it "links native video and audio players to the file they play" do
+ html = 'Clip:
' \
+ 'Your browser does not support video. ' \
+ ' '
+ convert(html).should == "Clip:\n\n[Demo clip](https://example.com/clip.mp4)\n\n[Video](https://example.com/b.webm)\n\n" \
+ "[Audio](https://example.com/a.mp3)"
+ end
+
+ it "escapes the title of a player like any text" do
+ convert('Listen: now
').should ==
+ "Listen:\n\n[Episode 1\\] \\ Live](https://example.com/ep.mp3)\n\nnow"
+ convert(' ').should ==
+ "[\\ \\*loud\\* \\[b\\\\](https://example.com/a.mp3)"
+ end
+
+ it "skips muted background videos, previews inside a link and players without a file" do
+ html = 'Hero
' \
+ ' Watch '
+ convert(html).should == "Hero\n\n[Watch](https://example.com/watch)"
+ end
+
+ it "keeps the thumbnail and link of a preview player inside a link" do
+ html = 'Watch film
'
+ convert(html).should == "Watch film\n\n[](https://example.com/watch)"
+ Nokodown.to_markdown(html, url: "https://example.com/").should ==
+ "Watch film\n\n[](https://example.com/watch)"
+ end
+
+ it "drops the thumbnail of a hidden preview player inside a link" do
+ ['hidden', 'aria-hidden="true"', 'style="display: none"', 'class="hidden"'].each do |hidden|
+ convert(%(Visible
)).should == "Visible"
+ convert(%(Visible
)).should == "Visible"
+ end
+ end
+
+ it "uses a of the player itself, not one of a player nested in it" do
+ convert(' ').should ==
+ "[Video](https://example.com/outer.mp4)"
+ end
+
+ it "finds the files of nested players in linear time" do
+ players = {
+ "nested" => "#{'' * 20_000}leaf#{' ' * 20_000}",
+ "in nested divs" => "#{' ' * 10_000}leaf#{'
' * 10_000}",
+ "in nested links" => "#{'' * 10_000}"
+ }
+ players.each_value do |html|
+ started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+ convert(html)
+ (Process.clock_gettime(Process::CLOCK_MONOTONIC) - started).should < 1
+ end
+ end
+
+ it "keeps a player and its caption in the main content" do
+ prose = "#{'Tidal lagoons hold water behind a wall at high tide, then release it through turbines. ' * 4}
"
+ html = "Lagoon #{prose}#{prose} "
+
+ Nokodown.to_markdown(html, url: "https://example.com/", title: false).should include(
+ "\n\n[Video](https://example.com/clip.mp4)\n\nRecorded demonstration.\n\n"
+ )
+ end
+
+ it "keeps players that sit with only links or other players" do
+ prose = "#{'Tidal lagoons hold water behind a wall at high tide, then release it through turbines. ' * 4}
"
+ html = "Lagoon #{prose}" \
+ "#{prose} "
+
+ Nokodown.to_markdown(html, url: "https://example.com/", title: false).should include(
+ "\n\n[Video](https://example.com/clip.mp4)\n\n[Download](https://example.com/clip.mp4)\n\n" \
+ "[Video](https://example.com/tide.mp4)\n\n[Audio](https://example.com/waves.mp3)\n\n"
+ )
+ end
+
+ it "still drops a player in furniture" do
+ prose = "#{'Tidal lagoons hold water behind a wall at high tide, then release it through turbines. ' * 4}
"
+ html = "Lagoon #{prose}#{prose} "
+
+ Nokodown.to_markdown(html, url: "https://example.com/", title: false).should_not include("promo")
+ end
+
it "uses the TeX source of MathML" do
math = 'x x^2 '
convert("Area #{math}
").should == "Area $x^2$"
end
+ it "writes raw TeX so that nothing in it reaches past it, and keeps its backslashes and braces" do
+ html = '[Run](javascript:alert(1))[
' \
+ ' ' \
+ '
'
+ convert(html).should == "$\\`$[`[Run](javascript:alert(1))[`](https://example.com/x)\n\n" \
+ "$\\frac{a}{b} \\{x\\} \\` 0\\Sam said:#{post}
Same here.
"
- convert(html).should ==
+ convert_content(html).should ==
"#{post}\n\n> Sam said:\n>\n> The stove boils a litre in three minutes on a full canister. …\n\nSame here."
end
@@ -666,13 +1092,13 @@ def code_block(code)
reply = "Mine takes four minutes. The wind matters more than the cold, so I always bring a windscreen along."
html = "#{post}
#{post}
#{reply}
" \
"Ann wrote:
#{post}
#{reply}
Agreed.
"
- convert(html).should == "#{post}\n\n> #{post}\n\n#{reply}\n\n> Ann wrote:\n>\n> #{reply} …\n\nAgreed."
+ convert_content(html).should == "#{post}\n\n> #{post}\n\n#{reply}\n\n> Ann wrote:\n>\n> #{reply} …\n\nAgreed."
end
it "closes the bold a shortened quote cuts through" do
post = "The stove boils a full litre of water in three minutes on a canister. It is a bit slower in the cold, " \
"and the piezo lighter fails above 3000 metres, so carry matches."
- convert("#{post}
#{post}
").should ==
+ convert_content("#{post}
#{post}
").should ==
"**#{post}**\n\n> **The stove boils a full litre of water in three minutes on a canister.** …"
end
@@ -705,16 +1131,55 @@ def code_block(code)
it "drops a section rendered a second time, but keeps a single repeated paragraph" do
first = "The stove boils a litre in three minutes on a full canister, and a little slower in cold mountain weather."
second = "It weighs 85 grams and folds into its own pot, so it takes almost no room at all in a pack or a bike bag."
- convert("").should == "#{first}\n\n#{second}"
- convert("#{first}
Day two.
#{first}
").should == "#{first}\n\nDay two.\n\n#{first}"
+ convert_content("").should == "#{first}\n\n#{second}"
+ convert_content("#{first}
Day two.
#{first}
").should == "#{first}\n\nDay two.\n\n#{first}"
+ end
+
+ it "drops the placeholder a script-loaded widget leaves, and its heading, from the main content" do
+ Nokodown.to_markdown("Fits a 1 litre pot.
Questions & Answers Loading Questions...
", title: false).should == "Fits a 1 litre pot."
+ end
+
+ it "drops a trailing heading left without content from the main content" do
+ Nokodown.to_markdown("Intro Text
Related ", title: false).should == "## Intro\n\nText"
+ end
+
+ it "drops boilerplate lines from the main content" do
+ Nokodown.to_markdown("Text
Advertisement
★★★
", title: false).should == "Text"
+ end
+
+ it "keeps headings without content when converting the whole page" do
+ convert("First
Second ").should == "First\n\n# Second"
+ convert("A B ").should == "## A\n\n## B"
+ convert("Intro Text
Related ").should == "## Intro\n\nText\n\n## Related"
+ end
+
+ it "keeps boilerplate lines and symbols but drops stray punctuation when converting the whole page" do
+ convert("Text
Advertisement
★★★
•
...
").should == "Text\n\nAdvertisement\n\n★★★"
+ end
+
+ it "keeps an image repeated elsewhere, and the symbols beside it, when converting the whole page" do
+ convert(' ★★★
').should ==
+ "\n\n\n★★★"
end
- it "drops the placeholder a script-loaded widget leaves, and its heading" do
- convert("Fits a 1 litre pot.
Questions & Answers Loading Questions...
").should == "Fits a 1 litre pot."
+ it "keeps everything selected but stray punctuation when converting the whole page" do
+ convert("Sample Download paper Visible dialog Closed ")
+ .should == "# Sample\n\nDownload paper\n\nVisible dialog"
+ convert("Download paper Visible dialog ").should == "Download paper\n\nVisible dialog"
+ convert('').should == "Note Prefilled text"
+ convert("Size Size Size ").should == "## Size\n\n## Size\n\n## Size"
+ convert("Mug Mug Cup ").should == "### Mug\n\n### Mug\n\n### Cup"
+ convert("First line
First line
Third line
").should == "First line\n\nFirst line\n\nThird line"
+ convert('').should ==
+ "\n\n"
+ convert(" ").should == "---"
+ convert("a
b
").should == "---\n\na\n\n---\n\n---\n\nb\n\n---"
+ convert("Text
•
...
").should == "Text"
end
- it "drops a trailing heading left without content" do
- convert("Intro Text
Related ").should == "## Intro\n\nText"
+ it "removes what a reader cannot see when converting the whole page" do
+ convert('Shown
Hidden
None
' \
+ 'Search ').should == "Shown\n\nSearch"
end
it "keeps a trailing list of linked titles" do
diff --git a/spec/nokodown/patterns_spec.rb b/spec/nokodown/patterns_spec.rb
new file mode 100644
index 0000000..331f8d5
--- /dev/null
+++ b/spec/nokodown/patterns_spec.rb
@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+
+RSpec.describe Nokodown::Patterns do
+ # Every string a pattern's branches spell out, for the patterns here:
+ # groups, alternation, "?" and "+", with lookarounds left out (they only
+ # narrow a match) and \d read as "1".
+ def spellings(source)
+ @source = source.gsub(/\s+/, "")
+ @at = 0
+ alternatives
+ end
+
+ def alternatives
+ found = sequence
+ while @source[@at] == "|"
+ @at += 1
+ found += sequence
+ end
+ found
+ end
+
+ def sequence
+ found = [""]
+ until @at >= @source.size || "|)".include?(@source[@at])
+ atom = self.atom
+ atom += [""] if @source[@at] == "?"
+ @at += 1 if "?+".include?(@source[@at].to_s)
+ found = found.product(atom).map(&:join)
+ end
+ found
+ end
+
+ def atom
+ char = @source[@at]
+ @at += 1
+ case char
+ when "\\" then (@at += 1) && [@source[@at - 1] == "d" ? "1" : @source[@at - 1]]
+ when "[" then (@at = @source.index("]", @at) + 1) && [@source[@at - 2]]
+ when "(" then group
+ else [char]
+ end
+ end
+
+ def group
+ lookaround = @source[@at, 3].match?(/\A\?[!=]/)
+ @at += lookaround ? 3 : 2
+ @at += 1 if lookaround && @source[@at - 1] == "<"
+ found = alternatives
+ @at += 1
+ lookaround ? [""] : found
+ end
+
+ # Boilerplate and the scorers check a name for these words before running
+ # the slow patterns, which only gives the same answer while every name a
+ # pattern matches holds one of the words.
+ { CHROME: :FURNITURE_HINT, NOISE: :FURNITURE_HINT, SECONDARY: :SECONDARY_HINT }.each do |pattern, hint|
+ it "finds a #{hint} word in every name #{pattern} matches" do
+ names = spellings(described_class.const_get(pattern).source)
+
+ names.size.should > 10
+ names.reject { |name| name.match?(described_class.const_get(pattern)) }.should == []
+ names.reject { |name| name.match?(described_class.const_get(hint)) }.should == []
+ end
+ end
+
+ it "answers furniture? and secondary? as the full patterns do" do
+ ["site-footer", "cookie-banner", "ad-slot", "share-bar", "post-body", "article-content", "comments", "tag-cloud", "tag-news"].each do |name|
+ described_class.furniture?(name).should == name.match?(described_class::PAGE_FURNITURE)
+ described_class.secondary?(name).should == name.match?(described_class::SECONDARY)
+ end
+ end
+
+ it "weighs hid as a class name of its own only" do
+ ["hid", "a hid", "hid b", "a hid b", "a\nhid", "hid\tb"].each { |value| value.should match(described_class::NEGATIVE) }
+ ["ahid", "hidx", "a-hid", "hid_b"].each { |value| value.should_not match(described_class::NEGATIVE) }
+ end
+end
diff --git a/spec/nokodown/scorer_spec.rb b/spec/nokodown/scorer_spec.rb
new file mode 100644
index 0000000..277cc79
--- /dev/null
+++ b/spec/nokodown/scorer_spec.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+
+RSpec.describe Nokodown::Scorer do
+ def score(html)
+ body = Nokolexbor::HTML("#{html}").at_css("body")
+ Nokodown::Dom.assign_keys(body)
+ described_class.new(body, Nokodown::Stats.new(body)).call
+ end
+
+ # Readability gives a paragraph one point, one per piece between commas,
+ # and one per 100 characters up to three.
+ it "counts every comma Readability counts, a trailing one too" do
+ result = score("Beans, roasted\u060C ground\uFE50 brewed\uFF0Cand served,
")
+
+ result.top.name.should == "article"
+ result.score.should == 1 + 6
+ end
+
+ it "scores each paragraph without listing its ancestors" do
+ paragraphs = Array.new(40) { "Coffee is brewed from roasted beans, ground fine.
" }.join
+ calls = 0
+ trace = TracePoint.new(:call) do |point|
+ calls += 1 if point.method_id == :ancestors && point.defined_class == Nokolexbor::Node
+ end
+
+ trace.enable { score("") }
+
+ calls.should < 10
+ end
+end
diff --git a/spec/nokodown/url_resolver_spec.rb b/spec/nokodown/url_resolver_spec.rb
new file mode 100644
index 0000000..3f26e69
--- /dev/null
+++ b/spec/nokodown/url_resolver_spec.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+RSpec.describe Nokodown::UrlResolver do
+ let(:urls) { described_class.new("https://example.com/dir/page?utm_source=x") }
+
+ it "gives the same answer when asked again for a link, an image or an anchor" do
+ 2.times do
+ urls.resolve("/a?utm_source=feed&id=1").should == "https://example.com/a?id=1"
+ urls.resolve_image("/img.jpg?w=300").should == "https://example.com/img.jpg"
+ urls.same_page?("https://example.com/dir/page#top").should == true
+ urls.same_page?("https://example.com/dir/other#top").should == false
+ urls.resolve(" ").should be_nil
+ end
+ end
+
+ it "reads a URL as a browser does, in linear time" do
+ described_class.normalize(" \x01 java\tscript:x\n \x7F").should == "javascript:x \x7F"
+ described_class.normalize(" \x05\t").should == ""
+
+ url = "/a#{' ' * 50_000}b"
+ started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+ described_class.normalize(" #{url} ").should == url
+ (Process.clock_gettime(Process::CLOCK_MONOTONIC) - started).should < 0.5
+ end
+
+ it "is not changed by changes to a string it was given or returned" do
+ href = +"/a"
+ first = urls.resolve(href)
+ href << "b"
+
+ first.should be_frozen
+ urls.resolve("/a").should == "https://example.com/a"
+ urls.resolve(href).should == "https://example.com/ab"
+ end
+end
diff --git a/spec/nokodown_spec.rb b/spec/nokodown_spec.rb
index e012854..4df7704 100644
--- a/spec/nokodown_spec.rb
+++ b/spec/nokodown_spec.rb
@@ -1,5 +1,7 @@
# frozen_string_literal: true
+require "zlib"
+
RSpec.describe Nokodown do
describe ".extract" do
it "returns the markdown with the page metadata" do
@@ -40,6 +42,10 @@
Nokodown.to_markdown(html).should == "# Is the ferry running today?"
end
+ it "keeps every heading of a page made of headings alone" do
+ Nokodown.to_markdown("One Two Three ").should == "# One\n\n## Two\n\n### Three"
+ end
+
it "puts the title above a section H1 that follows the opening paragraphs" do
intro = "#{'Vertical mice and trackballs spare the wrist in different ways, and we tested both kinds. ' * 3}
"
section = "#{'The first pick suits most hands and works on any desk surface we tried. ' * 3}
"
@@ -49,6 +55,16 @@
Nokodown.to_markdown(html).should include("\n\n## Deeper Dive: Our Picks\n\n")
end
+ it "keeps the page's headline above an article body chosen inside main" do
+ long = "Meaningful sentence with several useful words inside it. " * 10
+ page = "Breadcrumb trail
" \
+ "#{long}
"
+
+ Nokodown.to_markdown("#{page}").should start_with("# Story\n\nMeaningful sentence")
+ Nokodown.to_markdown("Story #{page}").should start_with("# Story\n\nMeaningful sentence")
+ Nokodown.to_markdown("Story #{page}", title: false).should start_with("Meaningful sentence")
+ end
+
it "keeps the page's own headline below a photo caption" do
body = "#{'Yamada earned its first star this year, and twelve more places joined the list. ' * 3}
"
html = "Michelin Restaurants in New York on Resy " \
@@ -221,6 +237,17 @@ def extract(html)
markdown.should include("- [Alan Turing](https://example.com/authors/alan)")
end
+ it "keeps the text of links that blocks cut into copies" do
+ html = "Title #{'Introduction, important body text. ' * 10}
"
+ markdown = extract(html)
+ markdown.should include("Details, useful content, and more.")
+ markdown.should include("Supplement, helpful text about this.")
+
+ html = "#{'Intro. ' * 80}#{'Lead. ' * 160}
#{'TAIL. ' * 80}"
+ extract(html).should include("TAIL. TAIL.")
+ end
+
it "keeps highlighted code whose tokens look like page furniture" do
html = "#{prose}print (x) #{prose} "
@@ -469,6 +496,134 @@ def band(topic)
end
end
+ describe "deeply nested markup" do
+ # Puma serves requests in threads, whose stack is much smaller than the
+ # main thread's.
+ def in_thread(&block)
+ Thread.new(&block).value
+ end
+
+ {
+ "blocks" => "Visible
#{'' * 3000}Deep#{'
' * 3000}",
+ "inline elements" => "Visible #{'' * 3000}Deep#{' ' * 3000}
",
+ "lists" => "Visible
#{'' * 1500}",
+ "links and emphasis" => "Visible
#{' ' * 1500}Deep#{' ' * 1500}",
+ "tables" => "Visible
#{'' * 1000}",
+ "noscript content" => "Hi
Visible #{'word ' * 100}
#{'' * 3000}Deep#{'
' * 3000} "
+ }.each do |shape, html|
+ it "renders #{shape} 3000 levels deep inside a thread" do
+ [true, false].each do |main_content|
+ in_thread { Nokodown.to_markdown(html, url: "https://example.com/", main_content: main_content) }.should include("Visible", "Deep")
+ end
+ end
+ end
+
+ it "renders 10,000 levels on the main thread" do
+ html = "#{%() * 10_000}deep#{'
' * 10_000}"
+
+ Nokodown.to_markdown(html).should == "deep"
+ Nokodown.to_markdown(("" * 4000) + "leaf" + ("
" * 4000)).should == "leaf"
+ end
+
+ it "keeps the words of flattened elements apart" do
+ html = "#{'' * 100}
One
Two Three
Four #{'
' * 100}"
+
+ convert(html).should == "One Two Three Four"
+ end
+
+ it "leaves the nesting of real pages alone" do
+ html = "#{'' * 60}"
+
+ convert(html).should == "## Title\n\nSome *text*"
+ end
+ end
+
+ describe "max_length:" do
+ def prose(index)
+ "Paragraph #{index} tells how the river rose through the night and the town woke to water in the streets and fields.
"
+ end
+
+ # The lengths at which max_length does not give the start of the whole
+ # conversion, or changes the excerpt.
+ def prefix_mismatches(html, lengths, **options)
+ whole = Nokodown.extract(html, url: "https://example.com/page", **options)
+ lengths.reject do |length|
+ cut = Nokodown.extract(html, url: "https://example.com/page", max_length: length, **options)
+ cut.markdown == whole.markdown[0, length] && cut.excerpt == whole.excerpt
+ end
+ end
+
+ # Pages long enough for rendering to stop early, each with something
+ # the whole content decides later on: a headline far down, a thumbnail
+ # shown again without its link, the excerpt, a list's spacing, a repeat
+ # that turns into a run, and a section label in an outline.
+ {
+ "a late H1" => ["Flood night %sFlood night %s ", {}],
+ "a thumbnail" => [" Story %s
%s
%s ", { title: false }],
+ "the excerpt" => ["%s%s", { main_content: false, title: false }],
+ "a loose list" => [" ", { main_content: false, title: false }],
+ "a repeated run" => ["%s
%s%s%s
%s%s ", { title: false }],
+ "an outline" => ["Outline %s%s ", { title: false }]
+ }.each do |name, (template, options)|
+ it "gives the start of the whole conversion of a page with #{name}" do
+ html = format(template, prose: Array.new(40) { |index| prose(index) }.join, one: prose(1), long: "x" * 1000, filler: "#{'z' * 90}
" * 12,
+ sentence: "The river rose through the night.", lines: Array.new(120) { |index| "Line #{index}
" }.join,
+ items: Array.new(120) { |index| "Item #{index} " }.join,
+ headings: Array.new(150) { |index| index == 3 ? "Tags " : "Heading #{index} " }.join)
+
+ prefix_mismatches(html, (0..3000).step(25), **options).should == []
+ end
+ end
+
+ it "gives the start of the whole conversion of each fixture" do
+ Dir[File.join(__dir__, "fixtures", "*.html")].sort.each do |path|
+ html = File.read(path) * 3
+ [true, false].each do |main_content|
+ [path, prefix_mismatches(html, [0, 1, 40, 300, 1000, 2500, 6000], main_content: main_content)].should == [path, []]
+ end
+ end
+ end
+
+ it "renders long content once, and only its start" do
+ html = "#{Array.new(2000) { |index| prose(index) }.join} "
+ renderers = []
+ Nokodown::Markdown::Renderer.should_receive(:new).once.and_wrap_original do |original, *args, **options|
+ original.call(*args, **options).tap { |renderer| renderers << renderer }
+ end
+
+ Nokodown.to_markdown(html, max_length: 500).length.should == 500
+ renderers.first.should be_cut
+ end
+
+ it "changes nothing when nil" do
+ html = "Flood night #{Array.new(40) { |index| prose(index) }.join} "
+
+ Nokodown.extract(html, max_length: nil).should == Nokodown.extract(html)
+ end
+
+ it "gives empty Markdown for 0" do
+ Nokodown.to_markdown("#{Array.new(40) { |index| prose(index) }.join} ", max_length: 0).should == ""
+ end
+
+ it "rejects a negative or non-integer length" do
+ -> { Nokodown.extract("x
", max_length: -1) }.should raise_error(ArgumentError)
+ -> { Nokodown.extract("x
", max_length: "50") }.should raise_error(ArgumentError)
+ end
+
+ # WCXB=tmp/corpora/wcxb bundle exec rspec checks every WCXB page.
+ it "gives the start of the whole conversion of every WCXB page" do
+ corpus = ENV["WCXB"]
+ skip "set WCXB to a WCXB checkout to run" unless corpus
+
+ Dir[File.join(corpus, "{dev,test}", "html", "*.html.gz")].sort.each do |path|
+ html = Zlib::GzipReader.open(path, &:read)
+ [true, false].each do |main_content|
+ [path, main_content, prefix_mismatches(html, [0, 1000, 5000, 50_000], main_content: main_content)].should == [path, main_content, []]
+ end
+ end
+ end
+ end
+
describe "encodings" do
it "transcodes binary input using the declared charset" do
html = "caf\xE9
".b
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index 3cda7ff..5b4df23 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -17,3 +17,12 @@ def fixture(name)
def convert(html, **options)
Nokodown.to_markdown("#{html}", url: "https://example.com/dir/page", main_content: false, title: false, **options)
end
+
+# Renders a fragment like #convert, then tidies it as main content is:
+# repeats, quotes of earlier posts, stray rules and images shown twice go.
+def convert_content(html)
+ body = Nokolexbor::HTML("#{html}").at_css("body")
+ Nokodown::Preprocessor.new(body).call
+ renderer = Nokodown::Markdown::Renderer.new(Nokodown::UrlResolver.new("https://example.com/dir/page"))
+ Nokodown::Markdown::Cleanup.call(renderer.render([body]), renderer.rendered_images).map(&:text).join("\n\n")
+end