feat: preload visited DNSLink URLs to local node (#827)

With #826
data was no longer getting into the cache of a local node 
if DNSLink redirect is disabled by the user.

This adds background queue that ensures every visited DNSLink
asset is preloaded to the cache of a local IPFS node, 
if the user chooses to do so.

Author: lidel

add-on/_locales/en/messages.json

@@ -287,6 +287,14 @@
     "message": "If global redirect is enabled, this will include DNSLink websites and redirect them to respective /ipns/{fqdn} paths at Custom Gateway",
     "description": "An option description on the Preferences screen (option_dnslinkRedirect_description)"
   },
+  "option_dnslinkDataPreload_title": {
+    "message": "Preload visited pages",
+    "description": "An option title on the Preferences screen (option_dnslinkDataPreload_title)"
+  },
+  "option_dnslinkDataPreload_description": {
+    "message": "DNSLink websites will be preloaded to the local IPFS node to enable offline access and improve resiliency against network failures",
+    "description": "An option description on the Preferences screen (option_dnslinkDataPreload_description)"
+  },
   "option_dnslinkRedirect_warning": {
     "message": "Redirecting to a path-based gateway breaks Origin-based security isolation of DNSLink website! Please leave this disabled unless you are aware of (and ok with) related risks.",
     "description": "A warning on the Preferences screen, displayed when URL does not belong to Secure Context (option_customGatewayUrl_warning)"

add-on/src/lib/dnslink.js

@@ -17,8 +17,11 @@ module.exports = function createDnslinkResolver (getState) {
   // DNSLink lookup result cache
   const cacheOptions = { max: 1000, maxAge: 1000 * 60 * 60 * 12 }
   const cache = new LRU(cacheOptions)
-  // upper bound for concurrent background lookups done by preloadDnslink(url)
-  const lookupQueue = new PQueue({ concurrency: 8 })
+  // upper bound for concurrent background lookups done by resolve(url)
+  const lookupQueue = new PQueue({ concurrency: 4 })
+  // preload of DNSLink data
+  const preloadUrlCache = new LRU(cacheOptions)
+  const preloadQueue = new PQueue({ concurrency: 4 })
 
   const dnslinkResolver = {
 
@@ -89,20 +92,34 @@ module.exports = function createDnslinkResolver (getState) {
       return dnslink
     },
 
-    // does not return anything, runs async lookup in the background
-    // and saves result into cache with an optional callback
-    preloadDnslink (url, cb) {
-      if (dnslinkResolver.canLookupURL(url)) {
-        lookupQueue.add(async () => {
-          const fqdn = new URL(url).hostname
-          const result = dnslinkResolver.readAndCacheDnslink(fqdn)
-          if (cb) {
-            cb(result)
-          }
-        })
-      } else if (cb) {
-        cb(null)
-      }
+    // runs async lookup in a queue in the background and returns the record
+    async resolve (url) {
+      if (!dnslinkResolver.canLookupURL(url)) return
+      const fqdn = new URL(url).hostname
+      const cachedResult = dnslinkResolver.cachedDnslink(fqdn)
+      if (cachedResult) return cachedResult
+      return lookupQueue.add(() => {
+        return dnslinkResolver.readAndCacheDnslink(fqdn)
+      })
+    },
+
+    // preloads data behind the url to local node
+    async preloadData (url) {
+      const state = getState()
+      if (!state.dnslinkDataPreload || state.dnslinkRedirect) return
+      if (preloadUrlCache.get(url)) return
+      preloadUrlCache.set(url, true)
+      const dnslink = await dnslinkResolver.resolve(url)
+      if (!dnslink) return
+      if (state.ipfsNodeType === 'embedded') return
+      if (state.peerCount < 1) return
+      return preloadQueue.add(async () => {
+        const { pathname } = new URL(url)
+        const preloadUrl = new URL(state.gwURLString)
+        preloadUrl.pathname = `${dnslink}${pathname}`
+        await fetch(preloadUrl.toString(), { method: 'HEAD' })
+        return preloadUrl
+      })
     },
 
     // low level lookup without cache

add-on/src/lib/ipfs-request.js

@@ -89,7 +89,7 @@ function createRequestModifier (getState, dnslinkResolver, ipfsPathValidator, ru
     if (request.type === 'main_frame') {
       // lazily trigger DNSLink lookup (will do anything only if status for root domain is not in cache)
       if (state.dnslinkPolicy && dnslinkResolver.canLookupURL(request.url)) {
-        dnslinkResolver.preloadDnslink(request.url)
+        dnslinkResolver.resolve(request.url) // no await: preload record in background
       }
     }
     return isIgnored(request.requestId)
@@ -142,15 +142,18 @@ function createRequestModifier (getState, dnslinkResolver, ipfsPathValidator, ru
           return redirectToGateway(request.url, state, ipfsPathValidator)
         }
         // Detect dnslink using heuristics enabled in Preferences
-        if (state.dnslinkRedirect && state.dnslinkPolicy && dnslinkResolver.canLookupURL(request.url)) {
-          const dnslinkRedirect = dnslinkResolver.dnslinkRedirect(request.url)
-          if (dnslinkRedirect && isSafeToRedirect(request, runtime)) {
-            // console.log('onBeforeRequest.dnslinkRedirect', dnslinkRedirect)
-            return dnslinkRedirect
+        if (state.dnslinkPolicy && dnslinkResolver.canLookupURL(request.url)) {
+          if (state.dnslinkRedirect) {
+            const dnslinkRedirect = dnslinkResolver.dnslinkRedirect(request.url)
+            if (dnslinkRedirect && isSafeToRedirect(request, runtime)) {
+              // console.log('onBeforeRequest.dnslinkRedirect', dnslinkRedirect)
+              return dnslinkRedirect
+            }
+          } else if (state.dnslinkDataPreload) {
+            dnslinkResolver.preloadData(request.url)
           }
           if (state.dnslinkPolicy === 'best-effort') {
-            // dnslinkResolver.preloadDnslink(request.url, (dnslink) => console.log(`---> preloadDnslink(${new URL(request.url).hostname})=${dnslink}`))
-            dnslinkResolver.preloadDnslink(request.url)
+            dnslinkResolver.resolve(request.url)
           }
         }
       }

add-on/src/lib/options.js

@@ -17,6 +17,7 @@ exports.optionDefaults = Object.freeze({
   automaticMode: true,
   linkify: false,
   dnslinkPolicy: 'best-effort',
+  dnslinkDataPreload: true,
   dnslinkRedirect: false,
   recoverFailedHttpRequests: true,
   detectIpfsPathHeader: true,

add-on/src/options/forms/dnslink-form.js

@@ -7,11 +7,13 @@ const switchToggle = require('../../pages/components/switch-toggle')
 
 function dnslinkForm ({
   dnslinkPolicy,
+  dnslinkDataPreload,
   dnslinkRedirect,
   onOptionChange
 }) {
   const onDnslinkPolicyChange = onOptionChange('dnslinkPolicy')
   const onDnslinkRedirectChange = onOptionChange('dnslinkRedirect')
+  const onDnslinkDataPreloadChange = onOptionChange('dnslinkDataPreload')
 
   return html`
     <form>
@@ -47,6 +49,15 @@ function dnslinkForm ({
             </option>
           </select>
         </div>
+        <div>
+          <label for="dnslinkDataPreload">
+            <dl>
+              <dt>${browser.i18n.getMessage('option_dnslinkDataPreload_title')}</dt>
+              <dd>${browser.i18n.getMessage('option_dnslinkDataPreload_description')}</dd>
+            </dl>
+          </label>
+          <div>${switchToggle({ id: 'dnslinkDataPreload', checked: dnslinkDataPreload, disabled: dnslinkRedirect, onchange: onDnslinkDataPreloadChange })}</div>
+        </div>
         <div>
           <label for="dnslinkRedirect">
             <dl>

add-on/src/options/page.js

@@ -80,6 +80,7 @@ module.exports = function optionsPage (state, emit) {
   })}
   ${dnslinkForm({
     dnslinkPolicy: state.options.dnslinkPolicy,
+    dnslinkDataPreload: state.options.dnslinkDataPreload,
     dnslinkRedirect: state.options.dnslinkRedirect,
     onOptionChange
   })}