From a85f7208bec9aa748325d901cfb363607bc8e728 Mon Sep 17 00:00:00 2001 From: Alaa Abdulridha Date: Wed, 28 Oct 2026 13:29:10 -0500 Subject: [PATCH 1/2] Ignore heading anchor ids in class/id rules Sites build heading ids from the heading's text ("h-what-is-social-media", "travel-related-coverage-2"), and docs generators put the same id on the section the heading opens. The furniture, secondary and negative-weight rules read words such as "social", "sponsor", "related" or "media" in those ids and dropped real headings and sections. Such ids are now marked once in the preprocessor and left out of signatures and cleaning weights, unless the heading is itself a junk label ("Related posts", "Feedback"). Co-Authored-By: Claude Opus 5.5 --- lib/nokodown.rb | 1 + lib/nokodown/dom.rb | 10 +++- lib/nokodown/heading_anchors.rb | 69 +++++++++++++++++++++++++++ lib/nokodown/preprocessor.rb | 1 + lib/nokodown/scorer.rb | 7 +-- spec/nokodown/boilerplate_spec.rb | 31 ++++++++++++ spec/nokodown/heading_anchors_spec.rb | 35 ++++++++++++++ spec/nokodown_spec.rb | 6 +++ 8 files changed, 156 insertions(+), 4 deletions(-) create mode 100644 lib/nokodown/heading_anchors.rb create mode 100644 spec/nokodown/heading_anchors_spec.rb diff --git a/lib/nokodown.rb b/lib/nokodown.rb index fadf7f7..ac4d1e3 100644 --- a/lib/nokodown.rb +++ b/lib/nokodown.rb @@ -7,6 +7,7 @@ require_relative "nokodown/dom" require_relative "nokodown/stats" require_relative "nokodown/patterns" +require_relative "nokodown/heading_anchors" require_relative "nokodown/tables" require_relative "nokodown/url_resolver" require_relative "nokodown/metadata" diff --git a/lib/nokodown/dom.rb b/lib/nokodown/dom.rb index d41f102..d63fa72 100644 --- a/lib/nokodown/dom.rb +++ b/lib/nokodown/dom.rb @@ -36,6 +36,9 @@ module Dom # survives the copy that article cleaning works on. THREAD = "data-nokodown-thread" + # Set by HeadingAnchors on elements whose id is made from their heading. + ANCHOR = "data-nokodown-anchor" + module_function def assign_keys(root) @@ -109,12 +112,17 @@ def squish(string) def signature(node) classes = node["class"] - id = node["id"] + id = name_id(node) return "" if classes.nil? && id.nil? signature_of("#{classes} #{id}") end + # The id as a name for the element; a heading anchor only repeats the heading. + def name_id(node) + node.key?(ANCHOR) ? nil : node["id"] + end + def signature_of(value) # Generated ids (Webflow "w-node-") would otherwise read as words. value = value.gsub(UUID, " ") if value.match?(UUID) diff --git a/lib/nokodown/heading_anchors.rb b/lib/nokodown/heading_anchors.rb new file mode 100644 index 0000000..bab6f1a --- /dev/null +++ b/lib/nokodown/heading_anchors.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +module Nokodown + # Sites give headings ids made from their text so tables of contents can + # link to them ("h-what-is-social-media", "related-coverage-2"), and + # documentation generators put the same id on the section the heading + # opens. Such an id says what the section is about, not what kind of + # section it is: "social" in it does not make a share bar. Marked once, so + # every class/id rule can leave it out. + module HeadingAnchors + PREFIX = /\A[a-z][-_]/.freeze + COUNTER = /[-_]\d+\z/.freeze + + # Headings that are nothing but the name of a widget: "Feedback", "Table + # of contents", "Customer reviews". + WIDGET_NAME = /\A(?:#{Patterns::PAGE_FURNITURE}|#{Patterns::SECONDARY}|#{Patterns::REVIEWS})\z/.freeze + + module_function + + def mark(root) + root.css("[id]").each { |node| node[Dom::ANCHOR] = "1" if anchor?(node) } + end + + def anchor?(node) + label = label(node) + return false if label.nil? || junk?(label) + + text = compact(label) + !text.empty? && variants(node["id"]).any? { |variant| compact(variant) == text } + end + + # A junk label's anchor names the widget under it, which goes too; keeping + # the heading would leave it without a section. + def junk?(label) + return true if label.match?(Patterns::NOISE_HEADING) || label.match?(Patterns::BOILERPLATE_TEXT) + + label.downcase.scan(/[\p{L}\p{N}]+/).join("-").match?(WIDGET_NAME) + end + + # The heading itself, a wiki's headline span inside it, or the heading + # that opens a section (after any empty anchor targets). + def label(node) + return Dom.text(node) if Dom.heading?(node) || Dom.heading?(node.parent) + + heading = opening_heading(node) + heading && Dom.text(heading) + end + + def opening_heading(node) + child = node.first_element_child + child = child.next_element while child && !Dom.heading?(child) && empty?(child) + child if child && Dom.heading?(child) + end + + def empty?(node) + node.first_element_child.nil? && node.content.strip.empty? + end + + # A one-letter prefix ("h-", "s-") and a counter for repeated headings. + def variants(id) + [id, id.sub(PREFIX, "")].flat_map { |variant| [variant, variant.sub(COUNTER, "")] } + end + + # Slugifiers differ in how they treat case, accents, apostrophes and spaces. + def compact(string) + string.unicode_normalize(:nfkd).downcase.gsub(/[^\p{L}\p{N}]/, "") + end + end +end diff --git a/lib/nokodown/preprocessor.rb b/lib/nokodown/preprocessor.rb index 6d7655e..284f7a2 100644 --- a/lib/nokodown/preprocessor.rb +++ b/lib/nokodown/preprocessor.rb @@ -52,6 +52,7 @@ def call @body.css("pre [class], table[class] td[class]").each { |node| node.remove if node["class"].downcase.match?(LINE_NUMBERS) } remove_hidden remove_form_controls + HeadingAnchors.mark(@body) Dom.assign_keys(@body) end diff --git a/lib/nokodown/scorer.rb b/lib/nokodown/scorer.rb index 1f0cd6a..d204f00 100644 --- a/lib/nokodown/scorer.rb +++ b/lib/nokodown/scorer.rb @@ -17,10 +17,11 @@ def self.class_weight(node) end # Conditional cleaning deletes whatever weighs negative, so it ignores CSS - # utility tokens. Candidate scoring keeps Readability's weight, which some - # pages need to pick the right candidate. + # utility tokens and heading anchors ("media" in "h-social-media-tips"). + # Candidate scoring keeps Readability's weight, which some pages need to + # pick the right candidate. def self.cleaning_weight(node) - [node["class"], node["id"]].sum do |value| + [node["class"], Dom.name_id(node)].sum do |value| tokens = value.to_s.downcase.split.reject { |token| utility_class?(token) } words_weight(tokens.join(" ")) end diff --git a/spec/nokodown/boilerplate_spec.rb b/spec/nokodown/boilerplate_spec.rb index 479fcfe..b6abbca 100644 --- a/spec/nokodown/boilerplate_spec.rb +++ b/spec/nokodown/boilerplate_spec.rb @@ -84,6 +84,37 @@ def prose(topic, count) end end + describe "ids made from a heading's text" do + let(:article) { "

Selling on social media

#{prose('Social selling', 4)}" } + + it "keeps headings and sections whose id is their heading's anchor" do + html = <<~HTML + #{article} +

What is social shopping?

#{prose('Social shopping', 1)} +

Sponsorship deals

#{prose('Sponsorship', 1)} +

Cookies

#{prose('Reading cookies', 1)}
+ HTML + + text = remaining(html) + text.should include("What is social shopping?") + text.should include("Sponsorship deals") + text.should include("Reading cookies is explained") + end + + it "still removes junk labels, ids that are not anchors and furniture classes" do + html = <<~HTML + #{article} + +

Share this article

Copy link
+ + + HTML + + text = remaining(html) + %w[Related Share Follow tips].each { |word| text.should_not include(word) } + end + end + describe "repeated modules" do def cards(classes, count) Array.new(count) do |index| diff --git a/spec/nokodown/heading_anchors_spec.rb b/spec/nokodown/heading_anchors_spec.rb new file mode 100644 index 0000000..59bd69e --- /dev/null +++ b/spec/nokodown/heading_anchors_spec.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +RSpec.describe Nokodown::HeadingAnchors do + def anchor?(html) + body = Nokolexbor::HTML("#{html}").at_css("body") + described_class.anchor?(body.at_css("[id]")) + end + + it "recognizes ids made from a heading's text, with a one-letter prefix or a counter" do + anchor?('

What’s Next for Social Shopping?

').should == true + anchor?('').should == true + anchor?('

Login shell

').should == true + anchor?('

Qué es el marketing social

').should == true + anchor?('

Related to Portage

').should == true + end + + it "recognizes the id of the section a heading opens" do + anchor?('

Cookies

Read them from the request.

').should == true + anchor?('

Sharing models

Import them.

').should == true + end + + it "does not take other ids for anchors" do + anchor?('').should == false + anchor?('

Latest posts.

Social feed

').should == false + anchor?('Social sharing').should == false + end + + it "does not take the id of a junk label for an anchor, since it names the widget under it" do + anchor?('').should == false + anchor?('

Share this article

').should == false + anchor?('

Feedback

').should == false + anchor?('

Table of contents

').should == false + anchor?('

Customer Reviews

').should == false + end +end diff --git a/spec/nokodown_spec.rb b/spec/nokodown_spec.rb index 48e8e9f..9621f89 100644 --- a/spec/nokodown_spec.rb +++ b/spec/nokodown_spec.rb @@ -314,6 +314,12 @@ def story(extra) end end + it "keeps a heading whose anchor holds a negative class word" do + markdown = story('

Media coverage of tidal lagoons

Newspapers mostly covered the cost.

') + + markdown.should include("## Media coverage of tidal lagoons") + end + it "still drops copies shown only at some screen sizes" do markdown = story('

Narrow screen copy of the intro.

') From a2e1c7859057ac022666c57265d5b4580acc34ea Mon Sep 17 00:00:00 2001 From: Alaa Abdulridha Date: Wed, 28 Oct 2026 13:58:05 -0500 Subject: [PATCH 2/2] Read junk heading labels without their permalink MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Heading-anchor markup adds a permalink ("¶", "#", a zero-width space) to the heading text, so "Related posts¶" or "Comments¶" slipped past the end-anchored junk patterns and their sections were kept. Match the patterns against the label's words only. Also treat any label that names reviews as junk, so a product page's "Ratings and Reviews" section is still removed, and accept an uppercase one-letter prefix ("H-Social-Tips"). Co-Authored-By: Claude Opus 5.5 --- lib/nokodown/heading_anchors.rb | 18 ++++++++++++------ spec/nokodown/boilerplate_spec.rb | 3 ++- spec/nokodown/heading_anchors_spec.rb | 9 +++++++++ spec/nokodown/product_reviews_spec.rb | 9 +++++++++ spec/nokodown_spec.rb | 7 +++++++ 5 files changed, 39 insertions(+), 7 deletions(-) diff --git a/lib/nokodown/heading_anchors.rb b/lib/nokodown/heading_anchors.rb index bab6f1a..68b6b6d 100644 --- a/lib/nokodown/heading_anchors.rb +++ b/lib/nokodown/heading_anchors.rb @@ -8,12 +8,14 @@ module Nokodown # section it is: "social" in it does not make a share bar. Marked once, so # every class/id rule can leave it out. module HeadingAnchors - PREFIX = /\A[a-z][-_]/.freeze + PREFIX = /\A[a-z][-_]/i.freeze COUNTER = /[-_]\d+\z/.freeze # Headings that are nothing but the name of a widget: "Feedback", "Table - # of contents", "Customer reviews". - WIDGET_NAME = /\A(?:#{Patterns::PAGE_FURNITURE}|#{Patterns::SECONDARY}|#{Patterns::REVIEWS})\z/.freeze + # of contents". Reviews are only removed from product pages, where a + # section headed by them is the reviews widget, so naming them anywhere + # will do: "Ratings and reviews". + WIDGET_NAME = /\A(?:#{Patterns::PAGE_FURNITURE}|#{Patterns::SECONDARY})\z|#{Patterns::REVIEWS}/.freeze module_function @@ -30,11 +32,15 @@ def anchor?(node) end # A junk label's anchor names the widget under it, which goes too; keeping - # the heading would leave it without a section. + # the heading would leave it without a section. Only its words count: the + # permalink the anchor markup adds ("¶", "#", a zero-width space) would + # keep the anchored patterns from matching. def junk?(label) - return true if label.match?(Patterns::NOISE_HEADING) || label.match?(Patterns::BOILERPLATE_TEXT) + words = label.downcase.scan(/[\p{L}\p{N}]+/) + phrase = words.join(" ") + return true if phrase.match?(Patterns::NOISE_HEADING) || phrase.match?(Patterns::BOILERPLATE_TEXT) - label.downcase.scan(/[\p{L}\p{N}]+/).join("-").match?(WIDGET_NAME) + words.join("-").match?(WIDGET_NAME) end # The heading itself, a wiki's headline span inside it, or the heading diff --git a/spec/nokodown/boilerplate_spec.rb b/spec/nokodown/boilerplate_spec.rb index b6abbca..76515ad 100644 --- a/spec/nokodown/boilerplate_spec.rb +++ b/spec/nokodown/boilerplate_spec.rb @@ -106,12 +106,13 @@ def prose(topic, count) #{article}

Share this article

Copy link
+ HTML text = remaining(html) - %w[Related Share Follow tips].each { |word| text.should_not include(word) } + %w[Related Share Older Follow tips].each { |word| text.should_not include(word) } end end diff --git a/spec/nokodown/heading_anchors_spec.rb b/spec/nokodown/heading_anchors_spec.rb index 59bd69e..0440fa9 100644 --- a/spec/nokodown/heading_anchors_spec.rb +++ b/spec/nokodown/heading_anchors_spec.rb @@ -10,6 +10,7 @@ def anchor?(html) anchor?('

What’s Next for Social Shopping?

').should == true anchor?('').should == true anchor?('

Login shell

').should == true + anchor?('

Social Tips

').should == true anchor?('

Qué es el marketing social

').should == true anchor?('

Related to Portage

').should == true end @@ -31,5 +32,13 @@ def anchor?(html) anchor?('

Feedback

').should == false anchor?('

Table of contents

').should == false anchor?('

Customer Reviews

').should == false + anchor?('

Ratings and Reviews

Sturdy and light.

').should == false + end + + it "reads a junk label without the permalink that anchor markup adds to it" do + anchor?('

Comments

Jane wrote: thanks.

').should == false + anchor?('').should == false + anchor?('

# Share this article

').should == false + anchor?(%()).should == false end end diff --git a/spec/nokodown/product_reviews_spec.rb b/spec/nokodown/product_reviews_spec.rb index 8d52af6..09d2187 100644 --- a/spec/nokodown/product_reviews_spec.rb +++ b/spec/nokodown/product_reviews_spec.rb @@ -42,6 +42,15 @@ def markdown(head, main) result.should_not include("summer of surveys") end + it "removes a reviews section whose id is made from its heading" do + anchored = '

Ratings and Reviews

It survived a whole summer of surveys ' \ + "in my back pocket, and the paper takes pencil and fountain pen alike.

" + result = markdown(product_type, "#{description}#{details}#{anchored}") + + result.should include("dot-grid paper") + result.should_not include("summer of surveys") + end + it "removes schema.org reviews even though each has a description of its own" do review = '

' \ "Bought three of these for my survey team and every one of them made it through a wet season of work.

" diff --git a/spec/nokodown_spec.rb b/spec/nokodown_spec.rb index 9621f89..eea5f6c 100644 --- a/spec/nokodown_spec.rb +++ b/spec/nokodown_spec.rb @@ -320,6 +320,13 @@ def story(extra) markdown.should include("## Media coverage of tidal lagoons") end + it "still drops comments whose heading carries a permalink" do + markdown = story('

Comments

Jane wrote: the turbine part was the clearest.

') + + markdown.should_not include("Comments") + markdown.should_not include("Jane wrote") + end + it "still drops copies shown only at some screen sizes" do markdown = story('

Narrow screen copy of the intro.

')