Create local cache for IB

Author: yashovardhan

.gitignore

@@ -19,3 +19,4 @@ npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
 .env
+.integrationBuilderCache

.prettierignore

@@ -7,3 +7,4 @@
 
 # Misc
 /files
+.integrationBuilderCache

.vscode/settings.json

@@ -43,5 +43,8 @@
   },
   "[javascript]": {
     "editor.defaultFormatter": "vscode.typescript-language-features"
+  },
+  "[typescriptreact]": {
+    "editor.defaultFormatter": "dbaeumer.vscode-eslint"
   }
 }

docusaurus.config.ts

@@ -188,7 +188,7 @@ const config: Config = {
   ],
   plugins: [
     path.resolve(__dirname, "plugins", "docusaurus-plugin-content-hub"),
-    [path.resolve(__dirname, "plugins", "docusaurus-plugin-virtual-files"), { rootDir: "files" }],
+    [path.resolve(__dirname, "plugins", "docusaurus-plugin-virtual-files"), { rootDir: ".integrationBuilderCache" }],
     path.resolve(__dirname, "plugins", "node-polyfills"),
     [
       path.resolve(__dirname, "plugins", "plugin-dynamic-route"),

package.json

@@ -74,6 +74,8 @@
     "eslint-plugin-simple-import-sort": "^10.0.0",
     "eslint-plugin-tsdoc": "^0.2.16",
     "husky": "^8.0.3",
+    "fs": "^0.0.1-security",
+    "glob": "^8.0.3",
     "lint-staged": "^14.0.1",
     "path-browserify": "^1.0.1",
     "prettier": "^3.1.0",

plugins/docusaurus-plugin-virtual-files/index.js

@@ -1,25 +1,66 @@
 const axios = require("axios");
+const util = require("util");
+const fs = require("fs");
+const path = require("path");
+const joi = require("joi");
+const readFileAsync = util.promisify(fs.readFile);
+const writeFileAsync = util.promisify(fs.writeFile);
+
+const environment = process.env.NODE_ENV || "development";
+
+async function fetchHostedFile(filename) {
+  try {
+    response = await axios.get("https:/raw.githubusercontent.com/" + filename);
+    var fileContent = response.data;
+    if (typeof fileContent !== "string") {
+      fileContent = JSON.stringify(fileContent, null, 2);
+    }
+    return fileContent;
+  } catch (e) {
+    console.log(`Error fetching ${filename}: ${e}`);
+    return "";
+  }
+}
+
 const hostedFileLinks = require("../../src/common/hostedFileLinks.json");
 module.exports = (context, options) => ({
   name: "docusaurus-plugin-virtual-files",
   async loadContent() {
+    const dir = path.resolve(context.siteDir, options.rootDir);
     const filenames = Object.values(hostedFileLinks);
     const fileContents = {};
 
-    for (const filename of filenames) {
-      var data;
-      try {
-        response = await axios.get(filename);
-        data = response.data;
-      } catch (e) {
-        data = "";
-        console.log(`Error fetching ${filename}: ${e}`);
+    if (environment === "development") {
+      var data = "";
+      for (const filename of filenames) {
+        const filePath = path.join(dir, filename.replaceAll("/", "-"));
+        const directoryPath = path.dirname(filePath);
+
+        try {
+          data = await readFileAsync(filePath);
+        } catch (e) {
+          console.log(`Fetching ${filename} since local cache not available`);
+
+          data = await fetchHostedFile(filename);
+
+          try {
+            if (!fs.existsSync(directoryPath)) {
+              fs.mkdirSync(directoryPath, { recursive: true });
+            }
+            await writeFileAsync(filePath, data);
+            console.log(`Saved ${filename} to cache`);
+          } catch (error) {
+            console.log(`Error saving ${filename} to cache`);
+          }
+        }
+        fileContents[filename] = data;
       }
-      if (typeof data !== "string") {
-        data = JSON.stringify(data, null, 2);
+    } else {
+      for (const filename of filenames) {
+        fileContents[filename] = await fetchHostedFile(filename);
       }
-      fileContents[filename] = data;
     }
+
     return fileContents;
   },
   async contentLoaded({ content, actions }) {
@@ -34,3 +75,11 @@ module.exports = (context, options) => ({
     });
   },
 });
+
+module.exports.validateOptions = ({ options, validate }) =>
+  validate(
+    joi.object({
+      rootDir: joi.string().required(),
+    }),
+    options
+  );