For migrating WordPress blog. Tabling, for re-rework.

renoirb · renoirb · commit 87d086132335 · 2020-03-02T11:35:49.000-05:00
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -0,0 +1,14 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Launch Program",
+      "program": "${workspaceFolder}/example.js"
+    }
+  ]
+}
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -0,0 +1,3 @@
+{
+  "git.ignoreLimitWarning": true
+}
diff --git a/.vscode/tasks.json b/.vscode/tasks.json
@@ -0,0 +1,12 @@
+{
+  // See https://go.microsoft.com/fwlink/?LinkId=733558
+  // for the documentation about the tasks.json format
+  "version": "2.0.0",
+  "tasks": [
+    {
+      "type": "npm",
+      "script": "start",
+      "problemMatcher": []
+    }
+  ]
+}
diff --git a/Makefile b/Makefile
@@ -17,11 +17,14 @@ test:
 
 .PHONY: archive
 archive:
-	yarn archive
+	bin/archivator archive
 
 .PHONY: analyze
 analyze:
-	yarn analyze
+	bin/archivator analyze
+
+markdownify:
+	yarn markdownify
 
 dist:
 	yarn dist
diff --git a/bin/archivator-markdownify.js b/bin/archivator-markdownify.js
@@ -0,0 +1,24 @@
+import {
+    catcher,
+    iterateIntoArchivable
+} from '../src/module';
+import {
+read,
+handle,
+write
+} from '../src/markdownify';
+
+const URL_LIST = 'archive/index.csv';
+
+(async () => {
+/**
+ * Something is going somewhat as an anti-pattern here.
+ * We want Promise.all(...) at each step, and it's not how
+ * it is as of now. Needs rework here. TODO
+ */
+  for (const archivable of iterateIntoArchivable(URL_LIST)) {
+    const contents = await read(archivable).catch(catcher);
+    const handled = await handle(contents).catch(catcher);
+    await write(handled).catch(catcher);
+  }
+})();
diff --git a/package.json b/package.json
@@ -2,37 +2,51 @@
   "name": "archivator",
   "version": "1.0.0",
   "description": "Fetch and stream body contents for URLs",
-  "repository": {
-    "type": "git",
-    "url": "https://github.com/renoirb/archivator.git"
-  },
   "bugs": {
     "url": "https://github.com/renoirb/archivator/issues"
   },
-  "bin": {
-    "archivator-archive": "./src/archive.js",
-    "archivator-analyze": "./src/analyze.js"
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/renoirb/archivator.git"
   },
+  "license": "MIT",
+  "author": "Renoir Boulanger <hello@renoirboulanger.com> (http://renoirb.com/)",
   "main": "index.js",
   "module": "module.js",
+  "bin": {
+    "archivator-analyze": "./dist/analyze.js",
+    "archivator-archive": "./dist/archive.js"
+  },
   "scripts": {
+    "analyze": "node_modules/.bin/babel-node src/analyze.js",
+    "archive": "node_modules/.bin/babel-node src/archive.js",
+    "dev": "node_modules/.bin/babel src/ -d dist/ -w -s",
     "dist": "node_modules/.bin/babel src/ --minified -d dist/ -s",
-    "test": "node_modules/.bin/mocha --compilers js:babel-core/register",
     "lint": "node_modules/.bin/xo",
-    "dev": "node_modules/.bin/babel src/ -d dist/ -w -s",
-    "archive": "node_modules/.bin/babel-node src/archive.js",
-    "analyze": "node_modules/.bin/babel-node src/analyze.js",
+    "lint:fix": "node_modules/.bin/xo --fix",
     "markdownify": "node_modules/.bin/babel-node src/markdownify.js",
-    "lint:fix": "node_modules/.bin/xo --fix"
+    "test": "node_modules/.bin/mocha --compilers js:babel-core/register"
+  },
+  "xo": {
+    "esnext": true,
+    "ignores": [
+      "dist/**",
+      "example.js"
+    ],
+    "plugins": [
+      "unicorn"
+    ],
+    "rules": {
+      "func-names": 0
+    },
+    "space": 2
   },
-  "author": "Renoir Boulanger <hello@renoirboulanger.com> (http://renoirb.com/)",
-  "license": "MIT",
   "dependencies": {
     "async-file": "^2.0.2",
     "babel-polyfill": "^6.23.0",
     "cheerio": "^0.22.0",
     "elasticsearch": "^12.1.3",
-    "esm": "^3.0.81",
+    "esm": "^3.2.25",
     "gen-readlines": "^0.1.3",
     "html-md-2": "^3.0.0",
     "node-fetch": "^1.6.3",
@@ -52,19 +66,5 @@
     "eslint-plugin-unicorn": "^1.0.0",
     "mocha": "^3.2.0",
     "xo": "^0.17.1"
-  },
-  "xo": {
-    "esnext": true,
-    "space": 2,
-    "plugins": [
-      "unicorn"
-    ],
-    "ignores": [
-      "dist/**",
-      "example.js"
-    ],
-    "rules": {
-      "func-names": 0
-    }
   }
 }
diff --git a/src/analyze.js b/src/analyze.js
@@ -7,22 +7,22 @@ import {
   cheerioLoad
 } from './common';
 
-import dictionary from './lists/stopwords.en';
-
-/**
- * https://www.ranks.nl/stopwords
- * http://xpo6.com/list-of-english-stop-words/
- */
-const stopWords = new Set(dictionary);
+import dictionary from './lists/stopwords.fr';
+import dictionaryEn from './lists/stopwords.en';
 
 function normalize(input) {
   const dto = String(input) || '';
   return dto.replace(/[^\w\s]|_/g, '').toLowerCase();
 }
 
 async function extractWords(recv, archivable) {
-  const loaded = cheerioLoad(recv);
-  return loaded.then(shard => {
+  /**
+   * https://www.ranks.nl/stopwords
+   * http://xpo6.com/list-of-english-stop-words/
+   */
+  const stopWordsSet = new Set([...dictionary, ...dictionaryEn]);
+
+  return cheerioLoad(recv).then(shard => {
     const truncate = archivable.truncate;
     shard(truncate).remove();
     const text = shard.text().split(' ');
@@ -31,7 +31,7 @@ async function extractWords(recv, archivable) {
     for (let i = 0; i < text.length; i++) {
       const word = normalize(text[i]);
       const withinCharRange = /^[a-zA-ZÀ-ÖØ-öø-ÿ]+$/.test(word);
-      const isNotStopWord = stopWords.has(word) === false;
+      const isNotStopWord = stopWordsSet.has(word) === false;
       const hasAtLeastTwo = word.length > 1;
       if (withinCharRange && isNotStopWord && hasAtLeastTwo) {
         if (foundOnce.has(word) === false) {
@@ -117,7 +117,7 @@ async function write(file, data = {}, boolOverwrite = true) {
 export default async archivable => {
   const slug = archivable.slug;
   const path = `archive/${slug}`;
-  const cacheFile = `${path}/document.html`;
+  const cacheFile = `${path}/cache.html`;
   const file = `${path}/analyze.json`;
   return Promise.resolve(cacheFile)
     .then(cacheFile => analyze(cacheFile, archivable))
diff --git a/src/common.js b/src/common.js
@@ -58,8 +58,10 @@ function readCachedError(errorObj) {
 }
 
 // Make possible to do extractLinks, markdownify, ... in parallel TODO
-async function cheerioLoad(recv, configObj = {}) {
-  return new Promise(resolve => resolve(cheerio.load(recv, configObj)));
+function cheerioLoad(recv, configObj = {}) {
+  // console.log('async cheerioLoad', { recv, configObj })
+  const loading = cheerio.load(recv, configObj);
+  return Promise.resolve(loading);
 }
 
 const urlNotInBlacklist = u => {
diff --git a/src/lists/stopwords.en.json b/src/lists/stopwords.en.json
@@ -563,5 +563,17 @@
   "yourself",
   "yourselves",
   "youve",
-  "zero"
-]
+  "zero",
+  "january",
+  "february",
+  "march",
+  "april",
+  "may",
+  "june",
+  "july",
+  "august",
+  "september",
+  "october",
+  "november",
+  "december"
+]
diff --git a/src/lists/stopwords.fr.json b/src/lists/stopwords.fr.json
@@ -0,0 +1,46 @@
+[
+  "a",
+  "avec",
+  "ce",
+  "dans",
+  "de",
+  "de",
+  "des",
+  "du",
+  "en",
+  "est",
+  "jai",
+  "je",
+  "la",
+  "le",
+  "les",
+  "long",
+  "mon",
+  "non",
+  "off",
+  "oh",
+  "ok",
+  "or",
+  "pas",
+  "possible",
+  "pour",
+  "que",
+  "qui",
+  "sensible",
+  "sont",
+  "sur",
+  "une",
+  "zero",
+  "janvier",
+  "fevrier",
+  "mars",
+  "avril",
+  "mai",
+  "juin",
+  "juillet",
+  "aout",
+  "septembre",
+  "octobre",
+  "novembre",
+  "decembre"
+]
diff --git a/src/markdownify.js b/src/markdownify.js
diff --git a/src/transformer.js b/src/transformer.js
diff --git a/yarn.lock b/yarn.lock

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+{`
	`2`	`+ "git.ignoreLimitWarning": true`
	`3`	`+}`
Original file line number	Diff line number	Diff line change
`@@ -58,8 +58,10 @@ function readCachedError(errorObj) {`
`58`	`58`	`}`
`59`	`59`
`60`	`60`	`// Make possible to do extractLinks, markdownify, ... in parallel TODO`
`61`		`-async function cheerioLoad(recv, configObj = {}) {`
`62`		`- return new Promise(resolve => resolve(cheerio.load(recv, configObj)));`
	`61`	`+function cheerioLoad(recv, configObj = {}) {`
	`62`	`+ // console.log('async cheerioLoad', { recv, configObj })`
	`63`	`+ const loading = cheerio.load(recv, configObj);`
	`64`	`+ return Promise.resolve(loading);`
`63`	`65`	`}`
`64`	`66`
`65`	`67`	`const urlNotInBlacklist = u => {`