Make into ESM. Review patterns.

Author: renoirb

bin/archivator

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+Function=Function//; node -r esm "$0" "$@"; exit
+
+/**
+ * Leverage Require ESM Node.js CLI option.
+ *
+ * Source https://github.com/nodejs/modules/issues/152#issuecomment-405657530
+ */
+
+// import path from 'path';
+
+const [
+  handler,
+  fileName,
+  ...args
+] = process.argv
+
+// const srcDir = path.resolve('../src')
+// module.paths.push(srcDir)
+
+import './archivator-archive'

bin/archivator-archive.js

@@ -0,0 +1,22 @@
+import {
+  catcher,
+  fetcher,
+  transformer,
+  analyzer,
+  iterateIntoArchivable
+} from '../src/module';
+
+const URL_LIST = 'archive/index.csv';
+
+(async () => {
+  /**
+   * Something is going somewhat as an anti-pattern here.
+   * We want Promise.all(...) at each step, and it's not how
+   * it is as of now. Needs rework here. TODO
+   */
+  for (const archivable of iterateIntoArchivable(URL_LIST)) {
+    await fetcher(archivable).catch(catcher);
+    await transformer(archivable).catch(catcher);
+    await analyzer(archivable).catch(catcher);
+  }
+})();

package.json

@@ -4,7 +4,7 @@
   "description": "Fetch and stream body contents for URLs",
   "repository": {
     "type": "git",
-    "url": "git@github.com:renoirb/archivator.git"
+    "url": "https://github.com/renoirb/archivator.git"
   },
   "bugs": {
     "url": "https://github.com/renoirb/archivator/issues"
@@ -13,6 +13,8 @@
     "archivator-archive": "./src/archive.js",
     "archivator-analyze": "./src/analyze.js"
   },
+  "main": "index.js",
+  "module": "module.js",
   "scripts": {
     "dist": "node_modules/.bin/babel src/ --minified -d dist/ -s",
     "test": "node_modules/.bin/mocha --compilers js:babel-core/register",
@@ -30,6 +32,7 @@
     "babel-polyfill": "^6.23.0",
     "cheerio": "^0.22.0",
     "elasticsearch": "^12.1.3",
+    "esm": "^3.0.81",
     "gen-readlines": "^0.1.3",
     "html-md-2": "^3.0.0",
     "node-fetch": "^1.6.3",

src/analyze.js

@@ -3,37 +3,33 @@
 import * as fsa from 'async-file';
 
 import {
-  readLines,
-  handleIndexSourceErrors,
-  readCached,
-  figureOutTruncateAndSelector,
+  readFileWithErrorHandling,
   cheerioLoad
 } from './common';
 
+import dictionary from './lists/stopwords.en';
+
 /**
  * https://www.ranks.nl/stopwords
  * http://xpo6.com/list-of-english-stop-words/
  */
-const stopWords = new Set(require('./stopwords'));
-
-const URL_LIST = 'archive/index.csv';
-const OVERWRITE = true;
+const stopWords = new Set(dictionary);
 
-function transformText(input) {
+function normalize(input) {
   const dto = String(input) || '';
   return dto.replace(/[^\w\s]|_/g, '').toLowerCase();
 }
 
-async function extractWords(recv, source) {
+async function extractWords(recv, archivable) {
   const loaded = cheerioLoad(recv);
   return loaded.then(shard => {
-    const {_, truncate} = figureOutTruncateAndSelector(source);
+    const truncate = archivable.truncate;
     shard(truncate).remove();
     const text = shard.text().split(' ');
     const words = Object.create(null);
     const foundOnce = new Set();
     for (let i = 0; i < text.length; i++) {
-      const word = transformText(text[i]);
+      const word = normalize(text[i]);
       const withinCharRange = /^[a-zA-ZÀ-ÖØ-öø-ÿ]+$/.test(word);
       const isNotStopWord = stopWords.has(word) === false;
       const hasAtLeastTwo = word.length > 1;
@@ -51,20 +47,18 @@ async function extractWords(recv, source) {
   });
 }
 
-async function read(source) {
-  const path = `archive/${source.slug}`;
-  const cacheFile = `${path}/cache.html`;
-  const targetFileName = `${path}/analyze.json`;
+async function analyze(cacheFile, archivable) {
   const cacheExists = await fsa.exists(cacheFile);
   const data = Object.create(null);
   if (cacheExists === true) {
-    const cacheData = await readCached(cacheFile);
-    const words = await extractWords(cacheData, source);
-    data.words = words;
+    const cacheData = await readFileWithErrorHandling(cacheFile);
+    const words = await extractWords(cacheData, archivable);
+    const {sorted, keywords} = await sortedAndKeywords(words);
+    data.words = Object.assign({}, sorted);
+    data.keywords = Object.assign({}, keywords);
   }
 
-  console.log(`\nProcessing ${path}`);
-  return {file: targetFileName, data};
+  return data;
 }
 
 function sort(subject = {}) {
@@ -82,52 +76,50 @@ function sort(subject = {}) {
   return sortable; // array in format [ [ key1, val1 ], [ key2, val2 ], ... ]
 }
 
-async function analyze(recv) {
-  const words = recv.data.words || {};
+async function sortedAndKeywords(words = {}) {
   const keywords = Object.create(null);
-  const sorted = sort(words);
+  const sorted = Object.create(null);
+  const sorting = sort(words);
   const max = 10;
   let iter = 0;
-  for (const popular of sorted) {
+  for (const popular of sorting) {
     const used = popular[1]; // word has been used n times
     const word = popular[0];
     if (iter <= max && used > 3) {
       keywords[word] = used;
     }
+    sorted[word] = used;
     iter++;
   }
 
   const wordCount = Object.keys(words).length;
-  const paddedCounter = String(wordCount).padStart('5', ' ');
-  let logLine = paddedCounter + ' words found';
+  let logLine = '  analysis:';
+  console.log(logLine);
+  logLine = '    words: ' + wordCount;
   console.log(logLine);
   const firstThreeKeywords = Object.keys(keywords).slice(0, 3).join(', ');
-  logLine = '      Top keywords: ' + firstThreeKeywords;
+  logLine = '    keywords: ' + firstThreeKeywords;
   console.log(logLine);
 
-  recv.data.keywords = keywords;
-
-  return recv;
+  return {sorted, keywords};
 }
 
-async function write({file, data = {}}, boolOverwrite = false) {
+async function write(file, data = {}, boolOverwrite = true) {
   const destExists = await fsa.exists(file);
+  const contents = JSON.stringify(data);
   if (destExists === false || (destExists === true && boolOverwrite)) {
-    await fsa.writeTextFile(file, JSON.stringify(data), 'utf8');
+    await fsa.writeTextFile(file, contents, 'utf8');
   }
 
-  return {file, data};
+  return {file, contents};
 }
 
-/**
- * Something is going somewhat as an anti-pattern here.
- * We want Promise.all(...) at each step, and it's not how
- * it is as of now. Needs rework here. TODO
- */
-for (const url of readLines(URL_LIST)) {
-  Promise.resolve(url)
-    .then(u => read(u))
-    .then(descriptor => analyze(descriptor))
-    .then(descriptor => write(descriptor, OVERWRITE))
-    .catch(handleIndexSourceErrors);
-}
+export default async archivable => {
+  const slug = archivable.slug;
+  const path = `archive/${slug}`;
+  const cacheFile = `${path}/document.html`;
+  const file = `${path}/analyze.json`;
+  return Promise.resolve(cacheFile)
+    .then(cacheFile => analyze(cacheFile, archivable))
+    .then(analyzed => write(file, analyzed));
+};

src/archivable.js

@@ -0,0 +1,130 @@
+import {URL} from 'url';
+
+import normalilzeUrlSlug from './normalizer/slugs';
+import {cheerioLoad, urlNotInBlacklist} from './common';
+import extractAssets from './assets-handler';
+
+function simplifyBody(truncate, selector, body) {
+  return cheerioLoad(body).then(shard => {
+    shard(truncate).remove();
+    return shard(selector);
+  });
+}
+
+async function extractLinks(body) {
+  const links = new Set();
+  return cheerioLoad(body).then(shard => {
+    shard('a[href]').each((_, element) => {
+      const href = shard(element).attr('href');
+      try {
+        const hrefObj = new URL(href);
+        const isHttpLink = hrefObj.protocol.startsWith('http');
+        if (isHttpLink) {
+          const rewritten = `${hrefObj.origin}${hrefObj.pathname}`;
+          if (urlNotInBlacklist(rewritten)) {
+            links.add(rewritten);
+          }
+        }
+      } catch (err) { }
+    });
+    return [...links];
+  });
+}
+
+function appendDefaultTruncate(truncateArg) {
+  // Truncate is to strip off any patterns we do not want
+  // as part of our archived article.
+  let truncate = (truncateArg.length === 0) ? '' : `${truncateArg},`;
+  truncate += 'script,style,noscript';
+  return truncate;
+}
+
+function appendDefaultSelector(selectorArg) {
+  // If we know exactly where the main content is, otherwise grab the whole
+  // document body.
+  return (selectorArg.length === 0) ? 'body' : `${selectorArg}`;
+}
+
+/**
+ * Given every row in source file .csv
+ * http://example.org/a/b.html;selector;truncate
+ *
+ * selector is the CSS selector where the main content is
+ * truncate is a list of CSS selectors to strip off
+ */
+class Archivable {
+  constructor(
+    url,
+    truncate = '',
+    selector = '',
+  ) {
+    if (typeof url === 'undefined') {
+      const message = `First argument is URL, and is Required`;
+      throw new Error(message);
+    }
+    this.url = url;
+    this.slug = normalilzeUrlSlug(url);
+    this.truncate = truncate;
+    this.selector = selector;
+    this.assets = [];
+    this.links = [];
+    this.body = '';
+  }
+
+  async setBody(body) {
+    const truncate = this.truncate;
+    const selector = this.selector;
+    const simplified = await simplifyBody(truncate, selector, body);
+    const html = simplified.html();
+    this.body = html;
+    const links = await extractLinks(html);
+    this.links = [...links];
+    const assets = await extractAssets(this);
+    this.assets = [...assets];
+  }
+
+  toJSON() {
+    const url = this.url;
+    const slug = this.slug;
+    const truncate = this.truncate;
+    const selector = this.selector;
+    const assets = [...this.assets];
+    const links = [...this.links];
+
+    return {
+      url,
+      slug,
+      truncate,
+      selector,
+      assets,
+      links
+    };
+  }
+
+  static fromJSON(arg) {
+    const argIsString = typeof arg === 'string';
+    if (argIsString === false) {
+      const message = 'Only String is supported';
+      throw new Error(message);
+    }
+    const {
+      url,
+      truncate,
+      selector
+    } = JSON.parse(arg);
+
+    return new Archivable(url, truncate, selector);
+  }
+
+  static fromLine(
+      line = 'http://localhost;;',
+  ) {
+    const [url, selectorArg = '', truncateArg = ''] = line.split(';');
+    const truncate = appendDefaultTruncate(truncateArg);
+    const selector = appendDefaultSelector(selectorArg);
+
+    return new Archivable(url, truncate, selector);
+  }
+}
+
+export default Archivable;