")
- return topNode
-
-getSiblingsContent = (doc, lang, currentSibling, baselinescoreSiblingsPara) ->
-
- if currentSibling[0].name == 'p' && currentSibling.text().length > 0
- return [currentSibling]
- else
- potentialParagraphs = currentSibling.find("p")
- if potentialParagraphs == null
- return null
- else
- ps = []
- potentialParagraphs.each () ->
- firstParagraph = doc(this)
- txt = firstParagraph.text()
-
- if txt.length > 0
- wordStats = stopwords(txt, lang)
- paragraphScore = wordStats.stopwordCount
- siblingBaselineScore = 0.30
- highLinkDensity = isHighlinkDensity(doc, firstParagraph)
- score = baselinescoreSiblingsPara * siblingBaselineScore
-
- if score < paragraphScore && !highLinkDensity
- ps.push(txt)
-
- return ps
-
-getSiblingsScore = (doc, topNode, lang) ->
- base = 100000
- paragraphsNumber = 0
- paragraphsScore = 0
- nodesToCheck = topNode.find("p")
-
- nodesToCheck.each () ->
- node = doc(this)
- textNode = node.text()
- wordStats = stopwords(textNode, lang)
- highLinkDensity = isHighlinkDensity(doc, node)
-
- if wordStats.stopwordCount > 2 && !highLinkDensity
- paragraphsNumber += 1
- paragraphsScore += wordStats.stopwordCount
-
- if paragraphsNumber > 0
- base = paragraphsScore / paragraphsNumber
-
- return base
-
-# Keep track of a node's score with a gravityScore attribute
-updateScore = (node, addToScore) ->
- currentScore = 0
- scoreString = node.attr('gravityScore')
- if scoreString
- currentScore = parseInt(scoreString)
-
- newScore = currentScore + addToScore
- node.attr("gravityScore", newScore)
-
-# Keep track of # of 'texty' child nodes under this node with
-# graveityNodes attribute
-updateNodeCount = (node, addToCount) ->
- currentScore = 0
- countString = node.attr('gravityNodes')
- if countString
- currentScore = parseInt(countString)
-
- newScore = currentScore + addToCount
- node.attr("gravityNodes", newScore)
-
-# Check the ratio of links to words in a node.
-# If the ratio is high, this node is probably trash.
-isHighlinkDensity = (doc, node) ->
- links = node.find('a')
- return false unless links.length > 0
-
- txt = node.text()
- words = txt.split(' ')
- numberOfWords = words.length
-
- sb = []
- links.each () ->
- sb.push(doc(this).text())
-
- linkText = sb.join(' ')
- linkWords = linkText.split(' ')
- numberOfLinkWords = linkWords.length
- numberOfLinks = links.length
- percentLinkWords = numberOfLinkWords / numberOfWords
- score = percentLinkWords * numberOfLinks
-
- score >= 1.0
-
-# Return a node's gravity score (amount of texty-ness under it)
-getScore = (node) ->
- grvScoreString = node.attr('gravityScore')
- if !grvScoreString
- return 0
- else
- parseInt(grvScoreString)
-
-
-isTableAndNoParaExist = (doc, e) ->
- subParagraphs = e.find("p")
-
- subParagraphs.each () ->
- p = doc(this)
- txt = p.text()
-
- if txt.length < 25
- doc(p).remove()
-
- subParagraphs2 = e.find("p")
- if subParagraphs2.length == 0 && !(e[0].name in ["td", "ul", "ol"])
- return true
- else
- return false
-
-isNodescoreThresholdMet = (doc, node, e) ->
- topNodeScore = getScore(node)
- currentNodeScore = getScore(e)
- thresholdScore = topNodeScore * 0.08
-
- if (currentNodeScore < thresholdScore) && !(e[0].name in ["td", "ul", "ol", "blockquote"])
- return false
- else
- return true
-
-# Remove any remaining trash nodes (clusters of nodes with little/no content)
-postCleanup = (doc, targetNode, lang) ->
- node = addSiblings(doc, targetNode, lang)
-
- node.children().each () ->
- e = doc(this)
- eTag = e[0].name
- if eTag not in ['p', 'a']
- if isHighlinkDensity(doc, e) || isTableAndNoParaExist(doc, e) || !isNodescoreThresholdMet(doc, node, e)
- doc(e).remove()
-
- return node
-
-cleanNull = (text) ->
- return text?.replace(/^null$/g, "")
-
-cleanText = (text) ->
- return text?.replace(/[\r\n\t]/g, " ").replace(/\s\s+/g, " ").replace(//g, "").replace(/�/g, "").trim()
-
-
-cleanTitle = (title, delimiters) ->
- titleText = title || ""
- usedDelimeter = false
- _.each delimiters, (c) ->
- if titleText.indexOf(c) >= 0 && !usedDelimeter
- titleText = biggestTitleChunk(titleText, c)
- usedDelimeter = true
- return cleanText(titleText)
-
-
-rawTitle = (doc) ->
- gotTitle = false
- titleText = ""
- # The first h1 or h2 is a useful fallback
- _.each [doc("meta[property='og:title']")?.first()?.attr("content"), \
- doc("h1[class*='title']")?.first()?.text(), \
- doc("title")?.first()?.text(), \
- doc("h1")?.first()?.text(), \
- doc("h2")?.first()?.text()], (candidate) ->
- if candidate && candidate.trim() && !gotTitle
- titleText = candidate.trim()
- gotTitle = true
-
- return titleText
diff --git a/src/formatter.coffee b/src/formatter.coffee
deleted file mode 100644
index 3fa5043..0000000
--- a/src/formatter.coffee
+++ /dev/null
@@ -1,124 +0,0 @@
-stopwords = require("./stopwords")
-_ = require("lodash")
-{XRegExp} = require('xregexp')
-
-module.exports = formatter = (doc, topNode, language) ->
- removeNegativescoresNodes(doc, topNode)
- linksToText(doc, topNode)
- addNewlineToBr(doc, topNode)
- replaceWithText(doc, topNode)
- removeFewwordsParagraphs(doc, topNode, language)
- return convertToText(doc, topNode)
-
-linksToText = (doc, topNode) ->
- nodes = topNode.find('a')
-
- nodes.each () ->
- doc(this).replaceWith(doc(this).html())
-
-ulToText = (doc, node) ->
- nodes = node.find('li')
- txt = ""
-
- nodes.each () ->
- txt = txt + "\n * #{doc(this).text()}"
-
- txt = txt + "\n"
- txt
-
-replaceWithText = (doc, topNode) ->
- nodes = topNode.find('b, strong, i, br, sup')
- nodes.each () ->
- doc(this).replaceWith(doc(this).text())
-
-cleanParagraphText = (rawText) ->
- txt = rawText.trim()
- txt.replace(/[\s\t]+/g, ' ')
- txt
-
-# Turn an html element (and children) into nicely formatted text
-convertToText = (doc, topNode) ->
- txts = []
- nodes = topNode.contents()
-
- # To hold any text fragments that end up in text nodes outside of
- # html elements
- hangingText = ""
-
- nodes.each () ->
- node = doc(this)
- nodeType = node[0].type
- nodeName = node[0].name
-
- # Handle top level text nodes by adding them to a running list
- # and then treating all the hanging nodes as one paragraph tag
- if nodeType == "text"
- hangingText += node.text()
- # Same as 'continue'
- return true
- else if nodeName == "ul"
- hangingText += ulToText(doc, node)
- return true
-
- # If we hit a real node and still have extra acculated text,
- # pop it out as if it was a paragraph tag
- if hangingText.length > 0
- txt = cleanParagraphText(hangingText)
- txts = txts.concat(txt.split(/\r?\n/))
- hangingText = ""
-
- txt = cleanParagraphText(node.text())
- txt = txt.replace(/(\w+\.)([A-Z]+)/, '$1 $2')
- txts = txts.concat(txt.split(/\r?\n/))
-
- # Catch any left-over hanging text nodes
- if hangingText.length > 0
- txt = cleanParagraphText(hangingText)
- txts = txts.concat(txt.split(/\r?\n/))
-
- txts = _.map txts, (txt) ->
- txt.trim()
-
- # Make sure each text chunk includes at least one text character or number.
- # This supports multiple languages words using XRegExp to generate the
- # regex that matches wranges of unicode characters used in words.
- regex = XRegExp('[\\p{Number}\\p{Letter}]')
- txts = _.filter txts, (txt) ->
- regex.test(txt)
-
- txts.join('\n\n')
-
-addNewlineToBr = (doc, topNode) ->
- brs = topNode.find("br")
- brs.each () ->
- br = doc(this)
- br.replaceWith("\n\n")
-
-# Remove nodes with a negative score because they are probably trash
-removeNegativescoresNodes = (doc, topNode) ->
- gravityItems = topNode.find("*[gravityScore]")
-
- gravityItems.each () ->
- item = doc(this)
- score = parseInt(item.attr('gravityScore')) || 0
-
- if score < 1
- doc(item).remove()
-
-# remove paragraphs that have less than x number of words,
-# would indicate that it's some sort of link
-removeFewwordsParagraphs = (doc, topNode, language) ->
- allNodes = topNode.find("*")
-
- allNodes.each () ->
- el = doc(this)
- tag = el[0].name
- text = el.text()
-
- stopWords = stopwords(text, language)
- if (tag != 'br' || text != '\\r') && stopWords.stopwordCount < 3 && el.find("object").length == 0 && el.find("embed").length == 0
- doc(el).remove()
- else
- trimmed = text.trim()
- if trimmed[0] == "(" && trimmed[trimmed.length - 1] == ")"
- doc(el).remove()
diff --git a/src/stopwords.coffee b/src/stopwords.coffee
deleted file mode 100644
index ecebd0c..0000000
--- a/src/stopwords.coffee
+++ /dev/null
@@ -1,47 +0,0 @@
-path = require('path')
-fs = require('fs')
-_ = require('lodash')
-
-cache = {}
-
-getFilePath = (language) ->
- path.join(__dirname, "..", "data", "stopwords", "stopwords-#{language}.txt")
-
-# Given a language, loads a list of stop words for that language
-# and then returns which of those words exist in the given content
-module.exports = stopwords = (content, language = 'en') ->
- filePath = getFilePath(language)
-
- if !fs.existsSync(filePath)
- console.error("WARNING: No stopwords file found for '#{language}' - defaulting to English!")
- filePath = getFilePath('en')
-
- if cache.hasOwnProperty(language)
- stopWords = cache[language]
- else
- stopWords = fs.readFileSync(filePath).toString().split('\n')
- .filter((s) -> s.length > 0)
- cache[language] = stopWords
-
- strippedInput = removePunctuation(content)
- words = candiateWords(strippedInput)
- overlappingStopwords = []
-
- count = 0
-
- _.each words, (w) ->
- count += 1
- if stopWords.indexOf(w.toLowerCase()) > -1
- overlappingStopwords.push(w.toLowerCase())
-
- {
- wordCount: count,
- stopwordCount: overlappingStopwords.length,
- stopWords: overlappingStopwords
- }
-
-removePunctuation = (content) ->
- content.replace(/[\|\@\<\>\[\]\"\'\.,-\/#\?!$%\^&\*\+;:{}=\-_`~()]/g,"")
-
-candiateWords = (strippedInput) ->
- strippedInput.split(' ')
diff --git a/src/unfluff.coffee b/src/unfluff.coffee
deleted file mode 100644
index e2cab08..0000000
--- a/src/unfluff.coffee
+++ /dev/null
@@ -1,122 +0,0 @@
-cheerio = require("cheerio")
-extractor = require("./extractor")
-cleaner = require("./cleaner")
-
-module.exports = unfluff = (html, language) ->
- doc = cheerio.load(html)
- lng = language || extractor.lang(doc)
-
- pageData =
- title: extractor.title(doc)
- softTitle: extractor.softTitle(doc)
- date: extractor.date(doc)
- author: extractor.author(doc)
- publisher: extractor.publisher(doc)
- copyright: extractor.copyright(doc)
- favicon: extractor.favicon(doc)
- description: extractor.description(doc)
- keywords: extractor.keywords(doc)
- lang: lng
- canonicalLink: extractor.canonicalLink(doc)
- tags: extractor.tags(doc)
- image: extractor.image(doc)
-
- # Step 1: Clean the doc
- cleaner(doc)
-
- # Step 2: Find the doc node with the best text
- topNode = extractor.calculateBestNode(doc, lng)
-
- # Step 3: Extract text, videos, images, links
- pageData.videos = extractor.videos(doc, topNode)
- pageData.links = extractor.links(doc, topNode, lng)
- pageData.text = extractor.text(doc, topNode, lng)
-
- pageData
-
-# Allow access to document properties with lazy evaluation
-unfluff.lazy = (html, language) ->
- title: () ->
- doc = getParsedDoc.call(this, html)
- @title_ ?= extractor.title(doc)
-
- softTitle: () ->
- doc = getParsedDoc.call(this, html)
- @softTitle_ ?= extractor.softTitle(doc)
-
- date: () ->
- doc = getParsedDoc.call(this, html)
- @date_ ?= extractor.date(doc)
-
- copyright: () ->
- doc = getParsedDoc.call(this, html)
- @copyright_ ?= extractor.copyright(doc)
-
- author: () ->
- doc = getParsedDoc.call(this, html)
- @author_ ?= extractor.author(doc)
-
- publisher: () ->
- doc = getParsedDoc.call(this, html)
- @publisher_ ?= extractor.publisher(doc)
-
- favicon: () ->
- doc = getParsedDoc.call(this, html)
- @favicon_ ?= extractor.favicon(doc)
-
- description: () ->
- doc = getParsedDoc.call(this, html)
- @description_ ?= extractor.description(doc)
-
- keywords: () ->
- doc = getParsedDoc.call(this, html)
- @keywords_ ?= extractor.keywords(doc)
-
- lang: () ->
- doc = getParsedDoc.call(this, html)
- @language_ ?= language or extractor.lang(doc)
-
- canonicalLink: () ->
- doc = getParsedDoc.call(this, html)
- @canonicalLink_ ?= extractor.canonicalLink(doc)
-
- tags: () ->
- doc = getParsedDoc.call(this, html)
- @tags_ ?= extractor.tags(doc)
-
- image: () ->
- doc = getParsedDoc.call(this, html)
- @image_ ?= extractor.image(doc)
-
- videos: () ->
- return @videos_ if @videos_?
- doc = getCleanedDoc.call(this, html)
- topNode = getTopNode.call(this, doc, this.lang())
- @videos_ = extractor.videos(doc, topNode)
-
- text: () ->
- return @text_ if @text_?
- doc = getCleanedDoc.call(this, html)
- topNode = getTopNode.call(this, doc, this.lang())
- @text_ = extractor.text(doc, topNode, this.lang())
-
- links: () ->
- return @links_ if @links_?
- doc = getCleanedDoc.call(this, html)
- topNode = getTopNode.call(this, doc, this.lang())
- @links_ = extractor.links(doc, topNode, this.lang())
-
-# Load the doc in cheerio and cache it
-getParsedDoc = (html) ->
- @doc_ ?= cheerio.load(html)
-
-# Cached version of calculateBestNode
-getTopNode = (doc, lng) ->
- @topNode_ ?= extractor.calculateBestNode(doc, lng)
-
-# Cached version of the cleaned doc
-getCleanedDoc = (html) ->
- return @cleanedDoc_ if @cleanedDoc_?
- doc = getParsedDoc.call(this, html)
- @cleanedDoc_ = cleaner(doc)
- @cleanedDoc_
diff --git a/test-setup.coffee b/test-setup.coffee
deleted file mode 100644
index 7f642b6..0000000
--- a/test-setup.coffee
+++ /dev/null
@@ -1,30 +0,0 @@
-path = require 'path'
-util = require 'util'
-fs = require 'fs'
-deepEqual = require 'deep-equal'
-
-global[name] = func for name, func of require 'assert'
-
-# See http://wiki.ecmascript.org/doku.php?id=harmony:egal
-egal = (a, b) ->
- if a is b
- a isnt 0 or 1/a is 1/b
- else
- a isnt a and b isnt b
-
-# A recursive functional equivalence helper; uses egal for testing equivalence.
-arrayEgal = (a, b) ->
- if egal a, b then yes
- else if (Array.isArray a) and Array.isArray b
- return no unless a.length is b.length
- return no for el, idx in a when not arrayEgal el, b[idx]
- yes
-
-global.inspect = (o) -> util.inspect o, no, 2, yes
-global.eq = (a, b, msg) -> ok egal(a, b), msg ? "#{inspect a} === #{inspect b}"
-global.arrayEq = (a, b, msg) -> ok arrayEgal(a, b), msg ? "#{inspect a} === #{inspect b}"
-global.deepEq = (a, b, msg) -> ok deepEqual(a, b), msg ? "#{inspect a} === #{inspect b}"
-
-global.fs = fs
-
-global[k] = v for own k, v of require './'
diff --git a/test/cleaner.coffee b/test/cleaner.coffee
deleted file mode 100644
index 772285e..0000000
--- a/test/cleaner.coffee
+++ /dev/null
@@ -1,109 +0,0 @@
-suite 'Cleaner', ->
- cleaner = require("../src/cleaner")
- cheerio = require("cheerio")
-
- test 'exists', ->
- ok cleaner
-
- test 'removes body classes', ->
- html = fs.readFileSync("./fixtures/test_businessWeek1.html").toString()
- origDoc = cheerio.load(html)
-
- eq origDoc("body").attr("class").trim(), "magazine"
-
- newDoc = cleaner(origDoc)
- eq newDoc("body").attr("class"), ''
-
- test 'removes article attrs', ->
- html = fs.readFileSync("./fixtures/test_gizmodo1.html").toString()
- origDoc = cheerio.load(html)
-
- eq origDoc("article").attr("class").trim(), "row post js_post_item status-published commented js_amazon_module"
-
- newDoc = cleaner(origDoc)
- eq newDoc("article").attr("class"), undefined
-
- test 'removes em tag from image-less ems', ->
- html = fs.readFileSync("./fixtures/test_gizmodo1.html").toString()
- origDoc = cheerio.load(html)
-
- eq origDoc("em").length, 6
-
- newDoc = cleaner(origDoc)
- eq newDoc("em").length, 0
-
- test 'removes scripts', ->
- html = fs.readFileSync("./fixtures/test_businessWeek1.html").toString()
- origDoc = cheerio.load(html)
-
- eq origDoc("script").length, 40
-
- newDoc = cleaner(origDoc)
- eq newDoc("script").length, 0
-
- test 'removes comments', ->
- html = fs.readFileSync("./fixtures/test_gizmodo1.html").toString()
- origDoc = cheerio.load(html)
- comments = origDoc('*').contents().filter () ->
- this.type == "comment"
- eq comments.length, 15
-
- newDoc = cleaner(origDoc)
- comments = newDoc('*').contents().filter () ->
- this.type == "comment"
- eq comments.length, 0
-
- test 'replaces childless divs with p tags', ->
- origDoc = cheerio.load("
text1
")
- newDoc = cleaner(origDoc)
- eq newDoc("div").length, 0
- eq newDoc("p").length, 1
- eq newDoc("p").text(), "text1"
-
- test 'replaces u tags with plain text', ->
- origDoc = cheerio.load("text1")
- newDoc = cleaner(origDoc)
- eq newDoc("u").length, 0
- eq newDoc("body").html(), "text1"
-
- test 'removes divs by re (ex: /caption/)', ->
- html = fs.readFileSync("./fixtures/test_aolNews.html").toString()
- origDoc = cheerio.load(html)
- captions = origDoc('div.caption')
- eq captions.length, 1
-
- newDoc = cleaner(origDoc)
- captions = newDoc('div.caption')
- eq captions.length, 0
-
- test 'removes naughty elms by re (ex: /caption/)', ->
- html = fs.readFileSync("./fixtures/test_issue28.html").toString()
- origDoc = cheerio.load(html)
- naughty_els = origDoc('.retweet')
- eq naughty_els.length, 2
-
- newDoc = cleaner(origDoc)
- naughty_els = newDoc('.retweet')
- eq naughty_els.length, 0
-
- test 'removes trash line breaks that wouldn\'t be rendered by the browser', ->
- html = fs.readFileSync("./fixtures/test_sec1.html").toString()
- origDoc = cheerio.load(html)
- newDoc = cleaner(origDoc)
-
- pEls = newDoc('p')
- cleanedParaText = pEls[9].children[0].data
- eq cleanedParaText.trim(), "“This transaction would not only strengthen our global presence, but also demonstrate our commitment to diversify and expand our U.S. commercial portfolio with meaningful new therapies,” said Russell Cox, executive vice president and chief operating officer of Jazz Pharmaceuticals plc. “We look forward to ongoing discussions with the FDA as we continue our efforts toward submission of an NDA for defibrotide in the U.S. Patients in the U.S. with severe VOD have a critical unmet medical need, and we believe that defibrotide has the potential to become an important treatment option for these patients.”"
-
- test 'inlines code blocks as test', ->
- html = fs.readFileSync("./fixtures/test_github1.html").toString()
- origDoc = cheerio.load(html)
- codeEls = origDoc('code')
- eq codeEls.length, 26
-
- newDoc = cleaner(origDoc)
- codeEls = newDoc('code')
- eq codeEls.length, 0
-
- # This is a code block that should still be present in the doc after cleaning
- ok newDoc('body').text().indexOf("extractor = require('unfluff');") > 0
diff --git a/test/cleaner.js b/test/cleaner.js
new file mode 100644
index 0000000..392cea9
--- /dev/null
+++ b/test/cleaner.js
@@ -0,0 +1,113 @@
+// Generated by CoffeeScript 1.12.6
+(function() {
+ suite('Cleaner', function() {
+ var cheerio, cleaner;
+ cleaner = require("../src/cleaner");
+ cheerio = require("cheerio");
+ test('exists', function() {
+ return ok(cleaner);
+ });
+ test('removes body classes', function() {
+ var html, newDoc, origDoc;
+ html = fs.readFileSync("./fixtures/test_businessWeek1.html").toString();
+ origDoc = cheerio.load(html);
+ eq(origDoc("body").attr("class").trim(), "magazine");
+ newDoc = cleaner(origDoc);
+ return eq(newDoc("body").attr("class"), '');
+ });
+ test('removes article attrs', function() {
+ var html, newDoc, origDoc;
+ html = fs.readFileSync("./fixtures/test_gizmodo1.html").toString();
+ origDoc = cheerio.load(html);
+ eq(origDoc("article").attr("class").trim(), "row post js_post_item status-published commented js_amazon_module");
+ newDoc = cleaner(origDoc);
+ return eq(newDoc("article").attr("class"), void 0);
+ });
+ test('removes em tag from image-less ems', function() {
+ var html, newDoc, origDoc;
+ html = fs.readFileSync("./fixtures/test_gizmodo1.html").toString();
+ origDoc = cheerio.load(html);
+ eq(origDoc("em").length, 6);
+ newDoc = cleaner(origDoc);
+ return eq(newDoc("em").length, 0);
+ });
+ test('removes scripts', function() {
+ var html, newDoc, origDoc;
+ html = fs.readFileSync("./fixtures/test_businessWeek1.html").toString();
+ origDoc = cheerio.load(html);
+ eq(origDoc("script").length, 40);
+ newDoc = cleaner(origDoc);
+ return eq(newDoc("script").length, 0);
+ });
+ test('removes comments', function() {
+ var comments, html, newDoc, origDoc;
+ html = fs.readFileSync("./fixtures/test_gizmodo1.html").toString();
+ origDoc = cheerio.load(html);
+ comments = origDoc('*').contents().filter(function() {
+ return this.type === "comment";
+ });
+ eq(comments.length, 15);
+ newDoc = cleaner(origDoc);
+ comments = newDoc('*').contents().filter(function() {
+ return this.type === "comment";
+ });
+ return eq(comments.length, 0);
+ });
+ test('replaces childless divs with p tags', function() {
+ var newDoc, origDoc;
+ origDoc = cheerio.load("
text1
");
+ newDoc = cleaner(origDoc);
+ eq(newDoc("div").length, 0);
+ eq(newDoc("p").length, 1);
+ return eq(newDoc("p").text(), "text1");
+ });
+ test('replaces u tags with plain text', function() {
+ var newDoc, origDoc;
+ origDoc = cheerio.load("text1");
+ newDoc = cleaner(origDoc);
+ eq(newDoc("u").length, 0);
+ return eq(newDoc("body").html(), "text1");
+ });
+ test('removes divs by re (ex: /caption/)', function() {
+ var captions, html, newDoc, origDoc;
+ html = fs.readFileSync("./fixtures/test_aolNews.html").toString();
+ origDoc = cheerio.load(html);
+ captions = origDoc('div.caption');
+ eq(captions.length, 1);
+ newDoc = cleaner(origDoc);
+ captions = newDoc('div.caption');
+ return eq(captions.length, 0);
+ });
+ test('removes naughty elms by re (ex: /caption/)', function() {
+ var html, naughty_els, newDoc, origDoc;
+ html = fs.readFileSync("./fixtures/test_issue28.html").toString();
+ origDoc = cheerio.load(html);
+ naughty_els = origDoc('.retweet');
+ eq(naughty_els.length, 2);
+ newDoc = cleaner(origDoc);
+ naughty_els = newDoc('.retweet');
+ return eq(naughty_els.length, 0);
+ });
+ test('removes trash line breaks that wouldn\'t be rendered by the browser', function() {
+ var cleanedParaText, html, newDoc, origDoc, pEls;
+ html = fs.readFileSync("./fixtures/test_sec1.html").toString();
+ origDoc = cheerio.load(html);
+ newDoc = cleaner(origDoc);
+ pEls = newDoc('p');
+ cleanedParaText = pEls[9].children[0].data;
+ return eq(cleanedParaText.trim(), "“This transaction would not only strengthen our global presence, but also demonstrate our commitment to diversify and expand our U.S. commercial portfolio with meaningful new therapies,” said Russell Cox, executive vice president and chief operating officer of Jazz Pharmaceuticals plc. “We look forward to ongoing discussions with the FDA as we continue our efforts toward submission of an NDA for defibrotide in the U.S. Patients in the U.S. with severe VOD have a critical unmet medical need, and we believe that defibrotide has the potential to become an important treatment option for these patients.”");
+ });
+ return test('inlines code blocks as test', function() {
+ var codeEls, html, newDoc, origDoc;
+ html = fs.readFileSync("./fixtures/test_github1.html").toString();
+ origDoc = cheerio.load(html);
+ codeEls = origDoc('code');
+ eq(codeEls.length, 26);
+ newDoc = cleaner(origDoc);
+ codeEls = newDoc('code');
+ eq(codeEls.length, 0);
+ return ok(newDoc('body').text().indexOf("extractor = require('unfluff');") > 0);
+ });
+ });
+
+}).call(this);
diff --git a/test/extractor.coffee b/test/extractor.coffee
deleted file mode 100644
index 3b2dd8e..0000000
--- a/test/extractor.coffee
+++ /dev/null
@@ -1,131 +0,0 @@
-suite 'Extractor', ->
- extractor = require("../src/extractor")
- cheerio = require("cheerio")
-
- test 'exists', ->
- ok extractor
-
- test 'returns a blank title', ->
- doc = cheerio.load("")
- title = extractor.title(doc)
- eq title, ""
-
- test 'returns a simple title', ->
- doc = cheerio.load("Hello!")
- title = extractor.title(doc)
- eq title, "Hello!"
-
- test 'returns a simple title chunk', ->
- doc = cheerio.load("This is my page - mysite")
- title = extractor.title(doc)
- eq title, "This is my page"
-
- test 'returns a soft title chunk without truncation', ->
- doc = cheerio.load("University Budgets: Where Your Fees Go | Top Universities")
- title = extractor.softTitle(doc)
- eq title, "University Budgets: Where Your Fees Go"
-
- test 'prefers the meta tag title', ->
- doc = cheerio.load("This is my page - mysite")
- title = extractor.title(doc)
- eq title, "Open graph title"
-
- test 'falls back to title if empty meta tag', ->
- doc = cheerio.load("This is my page - mysite")
- title = extractor.title(doc)
- eq title, "This is my page"
-
- test 'returns another simple title chunk', ->
- doc = cheerio.load("coolsite.com: This is my page")
- title = extractor.title(doc)
- eq title, "This is my page"
-
- test 'returns a title chunk without �', ->
- doc = cheerio.load("coolsite.com: � This� is my page")
- title = extractor.title(doc)
- eq title, "This is my page"
-
- test 'returns the first title;', ->
- doc = cheerio.load("This is my page")
- title = extractor.title(doc)
- eq title, "This is my page"
-
- test 'handles missing favicons', ->
- doc = cheerio.load("")
- favicon = extractor.favicon(doc)
- eq undefined, favicon
-
- test 'returns the article published meta date', ->
- doc = cheerio.load("")
- date = extractor.date(doc)
- eq date, "2014-10-15T00:01:03+00:00"
-
- test 'returns the article dublin core meta date', ->
- doc = cheerio.load("")
- date = extractor.date(doc)
- eq date, "2014-10-15T00:01:03+00:00"
-
- test 'returns the date in the