Remove feeds that dont work, or repeatedly timeout

Author: ojkelly

.vscode/settings.json

@@ -43,5 +43,8 @@
   "typescript.format.insertSpaceAfterKeywordsInControlFlowStatements": false,
   "typescript.tsc.autoDetect": "on",
   "typescript.validate.enable": true,
-  "markdown.extension.toc.githubCompatibility": true
+  "markdown.extension.toc.githubCompatibility": true,
+  "editor.codeActionsOnSave": {
+    "source.fixAll.eslint": true
+  }
 }

feeds/process.ts

@@ -2,7 +2,7 @@ import * as fs from "fs";
 import * as path from "path";
 import * as opmlToJSON from "opml-to-json";
 import * as rimraf from "rimraf";
-
+import { timeout } from "promise-timeout";
 import * as Parser from "rss-parser";
 
 import { OPML } from "./opml";
@@ -39,24 +39,64 @@ async function Process({ source, output }: { source: string; output: string }) {
           folder.children &&
           folder.children.map(async feed => {
             try {
-              const url = new URL(feed.htmlurl);
+              await timeout(
+                new Promise(async (resolve, reject) => {
+                  try {
+                    const url = new URL(feed.htmlurl);
+
+                    const items = await parser.parseURL(feed.xmlurl);
 
-              const items = await parser.parseURL(feed.xmlurl);
+                    await fs.mkdirSync(
+                      path.join(__dirname, output, url.hostname),
+                      {
+                        recursive: true,
+                      }
+                    );
 
-              await fs.mkdirSync(path.join(__dirname, output, url.hostname), {
-                recursive: true,
-              });
+                    await fs.writeFileSync(
+                      path.join(__dirname, output, url.hostname, "feed.json"),
+                      JSON.stringify({ ...feed, ...items }, null, 2),
+                      { encoding: "utf8" }
+                    );
 
-              await fs.writeFileSync(
-                path.join(__dirname, output, url.hostname, "feed.json"),
-                JSON.stringify({ ...feed, ...items }, null, 2),
-                { encoding: "utf8" }
+                    resolve();
+                  } catch (err) {
+                    reject(err);
+                  }
+                }),
+                60000
               );
               return true;
             } catch (err) {
-              // console.error(err);
-              // If this blog continues to error, we'll remove it
-              console.error("FAILED: ", feed.xmlurl);
+              if (!err.toString().includes("429")) {
+                console.error("FAILED: ", feed.xmlurl);
+
+                console.error(err);
+                // If this blog continues to error, we'll remove it
+                const opmlRaw = fs.readFileSync(path.join(__dirname, source));
+
+                const opmlLines = opmlRaw.toString().split("\n");
+
+                const opmlUpdate = opmlLines
+                  .map(line => {
+                    if (line.includes(feed.xmlurl)) {
+                      return;
+                    }
+
+                    return line;
+                  })
+                  .filter(Boolean)
+                  .join("\n");
+
+                await fs.writeFileSync(
+                  path.join(__dirname, source),
+                  opmlUpdate,
+                  {
+                    encoding: "utf8",
+                  }
+                );
+              }
+
               return true;
             }
           })

package.json

@@ -68,7 +68,7 @@
     "prettier": "^1.15.3",
     "react-scripts-ts": "3.1.0",
     "redbox-react": "^1.6.0",
-    "rss-parser": "^3.7.2",
+    "rss-parser": "^3.8.0",
     "spectacle-renderer": "^0.0.3",
     "ts-node": "^8.3.0",
     "typescript": "^3.3.3333",
@@ -132,6 +132,7 @@
     "opml-to-json": "^0.0.3",
     "prism-react-renderer": "^0.1.6",
     "prismjs": "^1.15.0",
+    "promise-timeout": "^1.3.0",
     "prop-types": "^15.6.2",
     "react": "^16.8.6",
     "react-dom": "^16.8.6",