refactor: remove webui TAR from Chromium bundle

We were bundling TAR inside of Chromium package to make it work
instantly when opened in Brave.

Unfortunately it added 10MB to the package, which meant one-click
install was slow in Brave, but also on every other Chromium browser.

This replaces bundled TAR with prefetching it from a public gateway on
the first run. The idea is that if user had access to Chrome Web Store
to install extension, they have access to the gateway, so no need for
bundling.

We also precache regular webui in Chromium and Firefox via ipfs.refs

Author: lidel

add-on/src/lib/ipfs-client/index.js

@@ -34,7 +34,7 @@ async function initIpfsClient (opts) {
 
   const instance = await client.init(opts)
   easeApiChanges(instance)
-  _reloadIpfsClientDependents(instance) // async (API is present)
+  _reloadIpfsClientDependents(instance, opts) // async (API is present)
   return instance
 }
 
@@ -71,9 +71,9 @@ async function _reloadIpfsClientDependents (instance, opts) {
     }
   }
   // online only
-  if (client && instance) {
+  if (client && instance && opts) {
     // add important data to local ipfs repo for instant load
-    setTimeout(() => precache(instance), 10000)
+    setTimeout(() => precache(instance, opts), 5000)
   }
 }
 

add-on/src/lib/precache.js

@@ -11,48 +11,78 @@ const debug = require('debug')
 const log = debug('ipfs-companion:precache')
 log.error = debug('ipfs-companion:precache:error')
 
-// Web UI release that should be precached
-// WARNING: do not remove this constant, as its used in package.json
-const webuiCid = 'bafybeigkbbjnltbd4ewfj7elajsbnjwinyk6tiilczkqsibf3o7dcr6nn4' // v2.9.0
-module.exports.precachedWebuiCid = webuiCid
-
+// TODO: make below smarter by detecting CID behind :5001/webui
+// (or webui.ipfs.io if bledding edge is enabled)
 const PRECACHE_ARCHIVES = [
-  { tarPath: '/dist/precache/webui.tar', cid: webuiCid }
+  {
+    nodeType: 'external',
+    name: 'webui v2.11.2',
+    cid: 'bafybeifekmcbbi4nwyj4aasti6x3nuhyli464wfjjfdjg4xnz53lhyiedq'
+  },
+  {
+    nodeType: 'embedded:chromesockets',
+    name: 'webui v2.9.0 for js-ipfs in Brave',
+    cid: 'bafybeigkbbjnltbd4ewfj7elajsbnjwinyk6tiilczkqsibf3o7dcr6nn4'
+  }
 ]
+module.exports.PRECACHE_ARCHIVES = PRECACHE_ARCHIVES
 
 /**
  * Adds important assets such as Web UI to the local js-ipfs-repo.
  * This ensures they load instantly, even in offline environments.
  */
-module.exports.precache = async (ipfs) => {
-  for (const { cid, tarPath } of PRECACHE_ARCHIVES) {
-    if (!await inRepo(ipfs, cid)) {
-      try {
-        const { body } = await fetch(tarPath)
-        log(`importing ${tarPath} to js-ipfs-repo`)
-        await importTar(ipfs, body.getReader(), cid)
-        log(`${tarPath} successfully cached under CID ${cid}`)
-      } catch (err) {
-        if (err.message === 'Error in body stream') {
-          // This error means .tar is missing from the extension bundle:
-          // It is the case in Firefox, due to https://github.com/ipfs-shipyard/ipfs-webui/issues/959
-          log(`unable to find/read ${tarPath}, skipping`)
+module.exports.precache = async (ipfs, state) => {
+  for (const { name, cid, nodeType } of PRECACHE_ARCHIVES) {
+    if (state.ipfsNodeType !== nodeType) continue
+    if (await inRepo(ipfs, cid)) {
+      log(`${name} (${cid}) already in local repo, skipping import`)
+      continue
+    }
+    log(`importing ${name} (${cid}) to js-ipfs-repo`)
+
+    // prefetch over HTTP when in Brave
+    if (state.ipfsNodeType === 'embedded:chromesockets') {
+      await preloadOverHTTP(ipfs, state, cid)
+      continue
+    }
+
+    // prefetch over IPFS
+    try {
+      for await (const ref of ipfs.refs(cid, { recursive: true })) {
+        if (ref.err) {
+          log.error(`error while preloading ${name} (${cid})`, ref.err)
           continue
         }
-        log.error(`error while processing ${tarPath}`, err)
       }
-    } else {
-      log(`${cid} already in local repo, skipping import`)
+      log(`${name} successfully cached under CID ${cid}`)
+    } catch (err) {
+      log.error(`error while processing ${name}`, err)
     }
   }
 }
 
 async function inRepo (ipfs, cid) {
-  for await (const ref of ipfs.refs.local()) {
-    if (ref.err) return false
-    if (ref.ref === cid) return true
+  // dag.get in offline mode will throw block is not present in local repo
+  // (we also have timeout as a failsafe)
+  try {
+    await ipfs.dag.get(cid, { offline: true, timeout: 5000 })
+    return true
+  } catch (_) {
+    return false
+  }
+}
+
+// Downloads CID from a public gateway
+// (alternative to ipfs.refs -r)
+async function preloadOverHTTP (ipfs, state, cid) {
+  try {
+    const url = `${state.pubGwURLString}api/v0/get?arg=${cid}&archive=true`
+    const { body } = await fetch(url)
+    await importTar(ipfs, body.getReader(), cid)
+    log(`successfully cached under CID ${cid}`)
+  } catch (err) {
+    log.error(`error while processing ${cid}`, err)
   }
-  return false
 }
 
 async function importTar (ipfs, tarReader, expectedCid) {
@@ -84,7 +114,7 @@ async function importTar (ipfs, tarReader, expectedCid) {
 
   const root = results.find(e => e.cid.toString(multibaseName) === expectedCid)
   if (!root) {
-    throw new Error('imported CID does not match expected one')
+    throw new Error(`imported CID (${root}) does not match expected one: ${expectedCid}`)
   }
 }
 

add-on/src/lib/state.js

@@ -2,7 +2,7 @@
 /* eslint-env browser, webextensions */
 
 const { safeURL } = require('./options')
-const { precachedWebuiCid } = require('./precache')
+const { PRECACHE_ARCHIVES } = require('./precache')
 const offlinePeerCount = -1
 
 function initState (options, overrides) {
@@ -48,12 +48,12 @@ function initState (options, overrides) {
       // Did user opt-in for rolling release published on DNSLink?
       if (state.useLatestWebUI) return `${state.gwURLString}ipns/webui.ipfs.io/`
 
-      // Below is needed to make webui work for embedded js-ipfs
-      // TODO: revisit if below is still needed after upgrading to js-ipfs >= 44
-      const webuiUrl = state.ipfsNodeType === 'embedded:chromesockets'
-        ? `${state.gwURLString}ipfs/${precachedWebuiCid}/`
-        : `${state.apiURLString}webui`
-      return webuiUrl
+      // Below is needed to make webui work for embedded js-ipfs in Brave
+      if (state.ipfsNodeType === 'embedded:chromesockets') {
+        const precached = PRECACHE_ARCHIVES.find(a => a.nodeType === 'embedded:chromesockets')
+        if (precached) return `${state.gwURLString}ipfs/${precached.cid}/`
+      }
+      return `${state.apiURLString}webui`
     }
   })
   // apply optional overrides

package.json

@@ -23,22 +23,20 @@
     "build:bundle-all": "cross-env RELEASE_CHANNEL=${RELEASE_CHANNEL:=dev} run-s bundle:chromium bundle:brave:$RELEASE_CHANNEL bundle:firefox:$RELEASE_CHANNEL",
     "build:rename-artifacts": "./scripts/rename-artifacts.js",
     "precache:clean": "shx rm -rf add-on/dist/precache",
-    "precache:webui:cid": "shx grep 'const webuiCid' add-on/src/lib/precache.js | shx sed \"s/^const webuiCid = '//\" | shx sed \"s/'.*$//\"",
-    "precache:webui": "shx mkdir -p add-on/dist/precache && ipfs-or-gateway -c $(npm run -s precache:webui:cid) -p add-on/dist/precache/webui.tar --archive",
     "bundle": "run-s bundle:*",
-    "bundle:chromium": "run-s precache:webui && shx cat add-on/manifest.common.json add-on/manifest.chromium.json | json --deep-merge > add-on/manifest.json && web-ext build -a build/chromium && run-s build:rename-artifacts",
-    "bundle:firefox": "run-s precache:clean && shx cat add-on/manifest.common.json add-on/manifest.firefox.json | json --deep-merge > add-on/manifest.json && web-ext build -a build/firefox/  && run-s build:rename-artifacts",
+    "bundle:chromium": "shx cat add-on/manifest.common.json add-on/manifest.chromium.json | json --deep-merge > add-on/manifest.json && web-ext build -a build/chromium && run-s build:rename-artifacts",
+    "bundle:firefox": "shx cat add-on/manifest.common.json add-on/manifest.firefox.json | json --deep-merge > add-on/manifest.json && web-ext build -a build/firefox/  && run-s build:rename-artifacts",
     "bundle:firefox:dev": "npm run bundle:firefox",
     "bundle:firefox:stable": "npm run bundle:firefox",
-    "bundle:firefox:beta": "run-s precache:clean && shx cat add-on/manifest.common.json add-on/manifest.firefox.json add-on/manifest.firefox-beta.json | json --deep-merge > add-on/manifest.json && web-ext build -a build/firefox/  && run-s build:rename-artifacts",
+    "bundle:firefox:beta": "shx cat add-on/manifest.common.json add-on/manifest.firefox.json add-on/manifest.firefox-beta.json | json --deep-merge > add-on/manifest.json && web-ext build -a build/firefox/  && run-s build:rename-artifacts",
     "bundle:fennec": "npm run bundle:firefox",
     "bundle:fennec:dev": "npm run bundle:firefox:dev",
     "bundle:fennec:stable": "npm run bundle:firefox:stable",
     "bundle:fennec:beta": "npm run bundle:firefox:beta",
-    "bundle:brave": "run-s precache:webui && shx cat add-on/manifest.common.json add-on/manifest.chromium.json add-on/manifest.brave.json | json --deep-merge > add-on/manifest.json && web-ext build -a build/brave/  && run-s build:rename-artifacts",
+    "bundle:brave": "shx cat add-on/manifest.common.json add-on/manifest.chromium.json add-on/manifest.brave.json | json --deep-merge > add-on/manifest.json && web-ext build -a build/brave/  && run-s build:rename-artifacts",
     "bundle:brave:dev": "npm run bundle:brave",
     "bundle:brave:stable": "npm run bundle:brave",
-    "bundle:brave:beta": "run-s precache:webui && shx cat add-on/manifest.common.json add-on/manifest.chromium.json add-on/manifest.brave.json add-on/manifest.brave-beta.json | json --deep-merge > add-on/manifest.json && web-ext build -a build/brave/  && run-s build:rename-artifacts",
+    "bundle:brave:beta": "shx cat add-on/manifest.common.json add-on/manifest.chromium.json add-on/manifest.brave.json add-on/manifest.brave-beta.json | json --deep-merge > add-on/manifest.json && web-ext build -a build/brave/  && run-s build:rename-artifacts",
     "watch": "npm-run-all build:copy --parallel watch:*",
     "watch:js": "run-p watch:js:*",
     "watch:js:webpack": "webpack --watch --progress -d --devtool inline-source-map  --config ./webpack.config.js",