diff --git a/README.md b/README.md index 9290a45..8065939 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ public API. Other classes are internal and can change in any release. | --- | --- | --- | | `url` | `nil` | Page URL. Relative links and images are resolved against it (and ``), and links to the page itself are reduced to their text. | | `main_content` | `true` | Keep only the main content. `false` converts the whole page, dropping only scripts, styles, hidden elements and form controls. | -| `title` | `true` | Start with the page title as a `#` heading when the content has no `

`. When no content is found at all (a page rendered by scripts, a bot wall), `markdown` is empty and only `result.title` has the title. | +| `title` | `true` | Start with the page title as a `#` heading when the content has no `

`, or when its first `

` follows the opening paragraphs and so heads a section. When no content is found at all (a page rendered by scripts, a bot wall), `markdown` is empty and only `result.title` has the title. | | `images` | `true` | Include images. Emoji images are always kept as their text. | | `links` | `true` | Keep links. `false` keeps only their text. | @@ -80,14 +80,23 @@ with `main_content: false`, `:none` for a document without a body. tables of contents, found through HTML semantics (`nav`, page-level `header` and `footer`, ARIA roles, schema.org types) and class/id names. - Cookie banners, ads, newsletter and sign-in prompts, share bars, modals, - chat widgets, "related", "recommended" and "most popular" boxes, and, next - to an article, its comment thread. + chat widgets, "related", "recommended" and "most popular" boxes, legal + disclaimers and disclosures (but not a note continuing the paragraph before + it, or the footnote to a mark just above), and, next to an article, its + comment thread. +- On forums, the member panel and signature repeated beside every post. The + member panel leaves one line naming the poster, with the post's date when + the panel holds it. A quote that repeats a post already in the output is cut + to its opening sentence, so the reply still shows what it answers. - On product pages, customer reviews and Q&A widgets, with their review summaries and rating histograms. An inline rating badge ("212 reviews" next to the price), testimonials and editorial reviews stay. - Invisible elements (`hidden`, `aria-hidden`, inline `display: none`, screen-reader-only and other hiding classes), keeping collapsed accordions, - tab panels and FAQ answers. + tab panels and FAQ answers. Content that React streams into hidden + containers at the end of the page is put back where the page shows it, + unless it is mostly links, as carousels of recommended products are. +- Sections rendered twice, such as a desktop and a mobile copy. - Tracking pixels, spacers, icons, placeholders, README badges, click-tracking URL parameters (`utm_*`, `gclid`, `fbclid`, ...), permalink anchors, "Read more" links, code line numbers, and repeated responsive copies of the diff --git a/lib/nokodown.rb b/lib/nokodown.rb index ac4d1e3..0b88420 100644 --- a/lib/nokodown.rb +++ b/lib/nokodown.rb @@ -12,6 +12,7 @@ require_relative "nokodown/url_resolver" require_relative "nokodown/metadata" require_relative "nokodown/preprocessor" +require_relative "nokodown/member_panel" require_relative "nokodown/boilerplate" require_relative "nokodown/product_reviews" require_relative "nokodown/scorer" diff --git a/lib/nokodown/boilerplate.rb b/lib/nokodown/boilerplate.rb index da8d9e8..9dce172 100644 --- a/lib/nokodown/boilerplate.rb +++ b/lib/nokodown/boilerplate.rb @@ -14,8 +14,18 @@ class Boilerplate PHRASING = Set.new(%w[a abbr b bdi bdo cite code data dfn em i kbd mark q s samp small span strong sub sup time u var]).freeze AD_ATTRIBUTES = %w[data-ad-slot data-ad-unit data-adunit data-ad-client data-google-query-id data-ad-format].freeze DRAWER = /(?= MIN_LEFT + + signature = Dom.signature(guess.first) + guess unless signature.match?(Patterns::PLACE) || signature.match?(Patterns::OVERLAY) + end + # Semantic chrome (nav, a page-level footer) goes even when it outweighs # the content, as legal footnotes often do; class-name guesses do not. - def remove?(node) - return false if @shielded.include?(Dom.key(node)) + def removal(node) + return if @shielded.include?(Dom.key(node)) # Side notes inside an article are content whatever their class says. - return false if node.name == "aside" && !aside_noise?(node) - return true if chrome_element?(node) || chrome_role?(node) - return false if dominant?(node) - return false unless advertising?(node) || chrome_label?(node) || class_noise?(node) + return if node.name == "aside" && !aside_noise?(node) + return :chrome if chrome_element?(node) || chrome_role?(node) + return if dominant?(node) + return unless advertising?(node) || chrome_label?(node) || class_noise?(node) + return if inside_sentence?(node) - !inside_sentence?(node) + MemberPanel.panel?(node) ? :panel : :guess end # A link or span in the middle of running text is part of that text, @@ -124,7 +152,55 @@ def chrome_label?(node) def class_noise?(node) return false if !@prose_blocks.empty? && prose_part?(node) - furniture_class?(node) && (@card_bases.empty? || !card?(node)) + furniture_class?(node) && !content_note?(node) && (@card_bases.empty? || !card?(node)) + end + + # Fine print about the site is furniture, but a note on the content right + # before it belongs to that content: a paragraph continuing the prose, or + # the footnote to a mark in it ("40-46 states*"). Widgets nest the note's + # text in an element named like the note. + def content_note?(node) + return false unless @disclaimer_only[Dom.signature(node)] + + note = @notes.include?(Dom.key(node.parent)) || annotates_previous?(node) + @notes << Dom.key(node) if note + note + end + + def annotates_previous?(node) + previous = node.previous_element + return false unless previous + + prose_note?(node, previous) || footnote_to?(node, previous) + end + + def prose_note?(node, previous) + return false unless node.name == "p" && previous.name == "p" + + @stats.text_chars(previous) >= NOTE_PROSE && !@furniture[Dom.signature(previous)] + end + + def footnote_to?(node, previous) + marks = [node, *node.css("p, li")].map { |line| Dom.text(line)[FOOTNOTE_MARK] }.compact.uniq + marks.any? { |mark| previous.content.match?(/[[:alnum:]]#{Regexp.escape(mark)}/) } + end + + # Only a signature that is furniture for its disclaimer name alone can be + # a note; "ad-disclaimer" or "footer__disclaimer" stays furniture. + def disclaimer_only?(signature) + signature.match?(Patterns::DISCLAIMER) && !signature.gsub(DISCLAIMER_WORD, " ").match?(Patterns::PAGE_FURNITURE) + end + + # Furniture for its signature name alone ("signature", "member-signature"). + def signature_only?(signature) + signature.match?(Patterns::SIGNATURE) && !signature.gsub(SIGNATURE_WORD, " ").match?(Patterns::PAGE_FURNITURE) + end + + # A member's signature is named after the post ("message-signature") or + # sits in one; a function's signature in API docs does neither. + def forum_signature?(node, signature) + signature.gsub(SIGNATURE_WORD, " ").match?(Patterns::POST) || + node.ancestors.any? { |ancestor| ancestor.element? && Dom.signature(ancestor).match?(Patterns::POST) } end def card?(node) @@ -148,7 +224,9 @@ def furniture_class?(node) return site_header?(node) if !chrome && signature.match?(Patterns::HEADER) return @stats.link_density(node) > 0.5 if !chrome && signature.match?(Patterns::MENU) return false unless chrome + return false if @signature_only[signature] && !forum_signature?(node, signature) return false if prose_section?(node, signature) + return true if signature.gsub(TAXONOMY, " ").match?(Patterns::NOISE) !(signature.match?(Patterns::POSITIVE) && @stats.text_chars(node) >= 200) end diff --git a/lib/nokodown/content_locator.rb b/lib/nokodown/content_locator.rb index ce0b44f..d01dd98 100644 --- a/lib/nokodown/content_locator.rb +++ b/lib/nokodown/content_locator.rb @@ -18,6 +18,11 @@ class ContentLocator REQUIRED_SHARE = { article: 0.15, product: 0.9 }.freeze DEFAULT_SHARE = 0.7 + # Sibling sections extend an article whose top candidate holds the share. + # When only they reach it, they must hold nearly the whole page: page mode + # would keep every band, and article mode drops the bands they leave out. + WHOLE_PAGE_SHARE = 0.9 + # Page types whose replies are content: forum threads and Q&A pages. DISCUSSIONS = %i[forum qa].freeze @@ -58,13 +63,21 @@ def main_region(stats) end def article?(result, root, stats) - article_text = result.nodes.sum { |node| stats.text_chars(node) } + required = REQUIRED_SHARE.fetch(@page_type, DEFAULT_SHARE) + return true if holds_share?(result.core, root, stats, required) + return false if result.nodes.size == result.core.size + + holds_share?(result.nodes, root, stats, [required, WHOLE_PAGE_SHARE].max) + end + + def holds_share?(nodes, root, stats, required) + article_text = nodes.sum { |node| stats.text_chars(node) } return false if article_text < MIN_ARTICLE_CHARS - page_text = stats.text_chars(root) - secondary_text(root, result.nodes, stats) + page_text = stats.text_chars(root) - secondary_text(root, nodes, stats) return true if page_text <= 0 - article_text / page_text >= REQUIRED_SHARE.fetch(@page_type, DEFAULT_SHARE) + article_text / page_text >= required end # Comment threads and "most popular" rails beside an article do not count diff --git a/lib/nokodown/dom.rb b/lib/nokodown/dom.rb index d63fa72..8eb420a 100644 --- a/lib/nokodown/dom.rb +++ b/lib/nokodown/dom.rb @@ -29,6 +29,9 @@ module Dom KEY = "data-nokodown-key" UUID = /\h{8}-\h{4}-\h{4}-\h{4}-\h{12}/.freeze + # One level of brackets may nest ("[&>[data-slot=icon]]"). Possessive, so + # a run of unclosed "[" is not rescanned from each one. + ARBITRARY_VALUE = /\[(?:[^\s\[\]]|\[[^\s\[\]]*+\])*+\]/.freeze CSS_MODULE_HASH = /__(?=[\w-]{0,4}[A-Z\d])[\w-]{5}(?![\w-])/.freeze # On a forum, the replies sit in containers named like a comment section. @@ -124,6 +127,8 @@ def name_id(node) end def signature_of(value) + # A Tailwind arbitrary value ("lg:h-[calc(100vh-var(--nav-height))]") is CSS, not a name. + value = value.gsub(ARBITRARY_VALUE, "") if value.include?("[") # Generated ids (Webflow "w-node-") would otherwise read as words. value = value.gsub(UUID, " ") if value.match?(UUID) value = value.gsub(CSS_MODULE_HASH, "") if value.include?("__") diff --git a/lib/nokodown/extractor.rb b/lib/nokodown/extractor.rb index 4d71c49..44a381b 100644 --- a/lib/nokodown/extractor.rb +++ b/lib/nokodown/extractor.rb @@ -3,6 +3,7 @@ module Nokodown class Extractor TITLE_SIMILARITY = 0.75 + OPENING_PROSE = 150 MIN_ARTICLE_LENGTH = 500 CONDITIONAL_SHARE = 0.5 REMOVED_LINK_SHARE = 0.5 @@ -90,21 +91,33 @@ def almost_nothing_kept?(copies, nodes) kept < MIN_ARTICLE_LENGTH && kept < nodes.sum { |node| Dom.text(node).length } * 0.25 end - # One H1 at the top: the page's own if the content has one, otherwise the - # metadata title, replacing a near-duplicate heading right below it. + # One H1 at the top: the page's own if the content opens with one, + # otherwise the metadata title, replacing a near-duplicate heading right + # below it. # Nothing found stays empty, so a script-rendered shell or a bot wall does # not pass for a page; Result#title still has the title. def with_title(blocks, title) return blocks if blocks.empty? first_h1 = blocks.find { |block| block.kind == :heading && block.level == 1 } - return demote_h1s(blocks, first_h1) if first_h1 + return demote_h1s(blocks, first_h1) if first_h1 && (title.nil? || page_heading?(blocks, first_h1, title)) return blocks if title.nil? duplicate = blocks.first(3).find do |block| block.kind == :heading && Dom.similarity(title, Markdown::Cleanup.heading_text(block)) > TITLE_SIMILARITY end - [Markdown::Block.new(:heading, "# #{Markdown::Text.escape(title)}", 1)] + blocks.reject { |block| block.equal?(duplicate) } + rest = demote_h1s(blocks.reject { |block| block.equal?(duplicate) }, nil) + [Markdown::Block.new(:heading, "# #{Markdown::Text.escape(title)}", 1)] + rest + end + + # When the page's title block is left out of the content, the first H1 + # left can be a section heading below the body's opening paragraphs. A + # headline may follow a caption or a promo line, but not the body. + def page_heading?(blocks, h1, title) + return true if Dom.similarity(title, Markdown::Cleanup.heading_text(h1)) > TITLE_SIMILARITY + + opening = blocks.take_while { |block| !block.equal?(h1) }.select { |block| block.kind == :paragraph } + opening.sum { |block| Markdown::Text.plain(block.text).length } < OPENING_PROSE end def demote_h1s(blocks, keep) diff --git a/lib/nokodown/markdown/cleanup.rb b/lib/nokodown/markdown/cleanup.rb index 5559168..cdc014f 100644 --- a/lib/nokodown/markdown/cleanup.rb +++ b/lib/nokodown/markdown/cleanup.rb @@ -10,6 +10,17 @@ module Cleanup IMAGE_ONLY = /\A!\[([^\]]*)\]\([^)]*\)\z/.freeze LINKED_HEADING = /\A#+ \[.*\]\(.*\)\z/.freeze TITLE_LIST_MIN = 3 + QUOTE_GRAM = 6 + REPEATED_BLOCK_MIN = 100 + REPEATED_RUN_MIN = 2 + REPEATED_QUOTE_SHARE = 0.5 + QUOTE_OPENING = 12 + # Not before a lowercase word or a number, as after "e.g." or "v.". + # Matched from the first mark of a run and without backtracking, so a long + # run of dots is not rescanned from each one on Ruby 2.7. + SENTENCE_END = /(?= REPEATED_RUN_MIN }.flatten.to_set + blocks.reject.with_index { |_, index| dropped.include?(index) } + end + + def repeatable?(block) + block.text.length >= REPEATED_BLOCK_MIN && block.kind != :heading && block.kind != :code + end + + # A reply that quotes a post shown earlier keeps the quote's opening, so + # the reader still sees which point it answers. Only posts count as the + # source: a quote like an earlier quote is the page's own repetition. + def shorten_repeated_quotes(blocks) + last_quote = blocks.rindex { |block| block.kind == :quote } + return blocks unless last_quote + + seen = Set.new + checked = blocks.first(last_quote + 1).map do |block| + grams = word_grams(block.text) + next(repeated?(grams, seen) ? quote_opening(block) : block) if block.kind == :quote + + seen.merge(grams) + block + end + checked + blocks.drop(last_quote + 1) + end + + def repeated?(grams, seen) + !grams.empty? && grams.count { |gram| seen.include?(gram) } >= grams.size * REPEATED_QUOTE_SHARE + end + + # A quote with fewer than QUOTE_OPENING words beyond its opening is cheap + # context and stays whole. The opening drops links and images: the post + # above has them. + def quote_opening(block) + lines = block.text.split("\n").map { |line| line.sub(/\A> ?/, "") } + own = lines.reject { |line| line.start_with?(">") }.take_while { |line| !line.start_with?("```", "~~~") } + opening = squeeze_blank(opening_lines(own.map { |line| Text.unlink(line).strip })) + return block if opening.empty? || words(block.text).size - words(opening.join(" ")).size < QUOTE_OPENING + + opening[-1] += " …" + Block.new(:quote, opening.map { |line| line.empty? ? ">" : "> #{line}" }.join("\n")) + end + + # The quoter's own lines up to the sentence that reaches QUOTE_OPENING + # words, or to the end of a paragraph that brings it to half of that. + # Lead-in lines ("Sam said:") do not count. + def opening_lines(lines) + count = 0 + lines.each_with_object([]) do |line, opening| + sentence = line.match?(SENTENCE_END) + next opening << line if line.end_with?(":") && !sentence + + size = words(line).size + break opening << sentence_head(line, QUOTE_OPENING - count) if count + size >= QUOTE_OPENING + break opening << line if sentence && count + size >= QUOTE_OPENING / 2 + + opening << line + count += size + end + end + + def squeeze_blank(lines) + lines = lines.chunk_while { |a, b| a.empty? && b.empty? }.map(&:first) + lines.shift while lines.first&.empty? + lines.pop while lines.last&.empty? + lines + end + + # The line up to the first sentence end after `needed` words, never + # inside brackets or code, closing bold it cuts; the whole line when + # there is none. The head grows a sentence at a time, so a line of many + # short sentences is read once rather than once per sentence. + def sentence_head(line, needed) + head = +"" + uncounted = +"" + size = open_brackets = ticks = 0 + line.scan(SENTENCE) do |sentence| + head << sentence + uncounted << sentence + open_brackets += sentence.count("[") - sentence.count("]") + ticks += sentence.count("`") + next unless open_brackets.zero? && ticks.even? + + # Counted once its brackets close, so a link that spans sentences is read whole. + size += words(uncounted).size + uncounted.clear + next if size < needed + + return head.scan("**").size.odd? ? "#{head}**" : head + end + line + end + + def word_grams(text) + words(text.downcase).each_cons(QUOTE_GRAM).map(&:hash) + end + + def words(text) + Text.plain(text).scan(/[[:word:]]+/) + end + def same_section(blocks) heading = blocks.rindex { |block| block.kind == :heading } heading ? blocks.drop(heading + 1) : blocks diff --git a/lib/nokodown/markdown/text.rb b/lib/nokodown/markdown/text.rb index 1e7d5a3..6e02ab0 100644 --- a/lib/nokodown/markdown/text.rb +++ b/lib/nokodown/markdown/text.rb @@ -64,11 +64,12 @@ def destination(url) # Markdown back to readable text: no images, link targets, emphasis or # escapes. def plain(markdown) - markdown.gsub(/!\[[^\]]*\]\([^)]*\)/, "") - .gsub(/\[([^\]]*)\]\([^)]*\)/, '\1') - .gsub(/\\([\\`*_\[\]<>#|])/, '\1') - .gsub(/\*\*|\*|~~|`/, "") - .strip + unlink(markdown).gsub(/\\([\\`*_\[\]<>#|])/, '\1').gsub(/\*\*|\*|~~|`/, "").strip + end + + # Markdown without images and link targets; link text stays. + def unlink(markdown) + markdown.gsub(/!\[[^\]]*\]\([^)]*\)/, "").gsub(/\[([^\]]*)\]\([^)]*\)/, '\1') end def table_cell(text) diff --git a/lib/nokodown/member_panel.rb b/lib/nokodown/member_panel.rb new file mode 100644 index 0000000..76b2c2a --- /dev/null +++ b/lib/nokodown/member_panel.rb @@ -0,0 +1,117 @@ +# frozen_string_literal: true + +module Nokodown + # Forum software prints a panel of member details beside every post: + # avatar, rank, join date, message and reaction counts. Repeated through a + # thread it is a sixth of the output, yet its name is all that says who + # wrote the post. The panel is reduced to one line, "**name**", with the + # post's date when the panel holds it ("answered Jan 4, 2024"). + module MemberPanel + HOLDERS = "[itemprop], [class]" + # "username", "username-coloured" (phpBB), "message-name" (XenForo), + # "cAuthorPane_author" (Invision). + NAME_CLASS = /user-?name|(?= text * STREAMED_LINK_SHARE + end + + # The fallback runs to the comment that closes the boundary, past any + # boundaries nested in it ("$", "$?" or "$!" open one, "/$" closes it). + def remove_fallback(slot) + fallback = [] + depth = 0 + node = slot.next_sibling + until node.nil? + if node.comment? && SUSPENSE_OPEN.include?(node.content) + depth += 1 + elsif node.comment? && node.content == "/$" + break if depth.zero? + + depth -= 1 + end + fallback << node + node = node.next_sibling + end + fallback.each(&:remove) if node + end + # Lexbor keeps