feat: recover dead sub-domain gateways (#802)

* add subdomainPublicGatewayUrl option with default of https://dweb.link
* add broken subdomain recovery via dweb.link public gateway
* add tests for subdomain recovery
* update gateway-form
* use subdomainPattern to extract protocol and cid from subdomain urls
* update add-on/_locales/en/messages.json

Co-Authored-By: Marcin Rataj <[email protected]>

Author: colinfruit

add-on/_locales/en/messages.json

@@ -291,6 +291,14 @@
     "message": "Fallback URL used when Custom Gateway is not available and for copying shareable links",
     "description": "An option description on the Preferences screen (option_publicGatewayUrl_description)"
   },
+  "option_publicSubdomainGatewayUrl_title": {
+    "message": "Default Public Subdomain Gateway",
+    "description": "An option title on the Preferences screen (option_publicSubdomainGatewayUrl_title)"
+  },
+  "option_publicSubdomainGatewayUrl_description": {
+    "message": "Default public subdomain gateway for recovery of broken subdomain gateways",
+    "description": "An option description on the Preferences screen (option_publicSubdomainGatewayUrl_description)"
+  },
   "option_header_api": {
     "message": "API",
     "description": "A section header on the Preferences screen (option_header_api)"

add-on/src/lib/ipfs-companion.js

@@ -685,6 +685,10 @@ module.exports = async function init () {
           state.pubGwURL = new URL(change.newValue)
           state.pubGwURLString = state.pubGwURL.toString()
           break
+        case 'publicSubdomainGatewayUrl':
+          state.pubSubdomainGwURL = new URL(change.newValue)
+          state.pubSubdomainGwURLString = state.pubSubdomainGwURL.toString()
+          break
         case 'useCustomGateway':
           state.redirect = change.newValue
           break

add-on/src/lib/ipfs-path.js

@@ -24,11 +24,13 @@ function subdomainToIpfsPath (url) {
   if (typeof url === 'string') {
     url = new URL(url)
   }
-  const fqdn = url.hostname.split('.')
+  const match = url.toString().match(IsIpfs.subdomainPattern)
+  if (!match) throw new Error('no match for IsIpfs.subdomainPattern')
+
   // TODO: support CID split with commas
-  const cid = fqdn[0]
+  const cid = match[1]
   // TODO: support .ip(f|n)s. being at deeper levels
-  const protocol = fqdn[1]
+  const protocol = match[2]
   return `/${protocol}/${cid}${url.pathname}${url.search}${url.hash}`
 }
 
@@ -38,6 +40,18 @@ function pathAtHttpGateway (path, gatewayUrl) {
 }
 exports.pathAtHttpGateway = pathAtHttpGateway
 
+function redirectSubdomainGateway (url, subdomainGateway) {
+  if (typeof url === 'string') {
+    url = new URL(url)
+  }
+  const match = url.toString().match(IsIpfs.subdomainPattern)
+  if (!match) throw new Error('no match for IsIpfs.subdomainPattern')
+  const cid = match[1]
+  const protocol = match[2]
+  return trimDoubleSlashes(`${subdomainGateway.protocol}//${cid}.${protocol}.${subdomainGateway.hostname}${url.pathname}${url.search}${url.hash}`)
+}
+exports.redirectSubdomainGateway = redirectSubdomainGateway
+
 function trimDoubleSlashes (urlString) {
   return urlString.replace(/([^:]\/)\/+/g, '$1')
 }
@@ -72,7 +86,11 @@ function createIpfsPathValidator (getState, getIpfs, dnslinkResolver) {
     validIpfsOrIpnsPath (path) {
       return validIpfsOrIpnsPath(path, dnslinkResolver)
     },
-
+    // Test if URL is a subdomain gateway resource
+    // TODO: add test if URL is a public subdomain resource
+    ipfsOrIpnsSubdomain (url) {
+      return IsIpfs.subdomain(url)
+    },
     // Test if actions such as 'copy URL', 'pin/unpin' should be enabled for the URL
     isIpfsPageActionsContext (url) {
       return Boolean(url && !url.startsWith(getState().apiURLString) && (
@@ -108,6 +126,17 @@ function createIpfsPathValidator (getState, getIpfs, dnslinkResolver) {
       // Return original URL (eg. DNSLink domains) or null if not an URL
       return input.startsWith('http') ? input : null
     },
+    // Resolve URL or path to subdomain gateway
+    // - non-subdomain path is returned as-is
+    // The purpose of this resolver is to return a valid IPFS
+    // subdomain URL
+    resolveToPublicSubdomainUrl (url, optionalGatewayUrl) {
+      // if non-subdomain return as-is
+      if (!IsIpfs.subdomain(url)) return url
+
+      const gateway = optionalGatewayUrl || getState().pubSubdomainGwURL
+      return redirectSubdomainGateway(url, gateway)
+    },
 
     // Resolve URL or path to IPFS Path:
     // - The path can be /ipfs/ or /ipns/

add-on/src/lib/ipfs-request.js

@@ -392,7 +392,13 @@ function createRequestModifier (getState, dnslinkResolver, ipfsPathValidator, ru
       // Check if error can be recovered by opening same content-addresed path
       // using active gateway (public or local, depending on redirect state)
       if (isRecoverable(request, state, ipfsPathValidator)) {
-        const redirectUrl = ipfsPathValidator.resolveToPublicUrl(request.url, state.pubGwURLString)
+        let redirectUrl
+        // if subdomain request redirect to default public subdomain url
+        if (ipfsPathValidator.ipfsOrIpnsSubdomain(request.url)) {
+          redirectUrl = ipfsPathValidator.resolveToPublicSubdomainUrl(request.url, state.pubSubdomainGwURL)
+        } else {
+          redirectUrl = ipfsPathValidator.resolveToPublicUrl(request.url, state.pubGwURLString)
+        }
         log(`onErrorOccurred: attempting to recover from network error (${request.error}) for ${request.url}`, redirectUrl)
         return createTabWithURL({ redirectUrl }, browser)
       }
@@ -404,13 +410,16 @@ function createRequestModifier (getState, dnslinkResolver, ipfsPathValidator, ru
       const state = getState()
       if (!state.active) return
       if (request.statusCode === 200) return // finish if no error to recover from
+      let redirectUrl
       if (isRecoverable(request, state, ipfsPathValidator)) {
-        const redirectUrl = ipfsPathValidator.resolveToPublicUrl(request.url, state.pubGwURLString)
-        const redirect = { redirectUrl }
-        if (redirect) {
-          log(`onCompleted: attempting to recover from HTTP Error ${request.statusCode} for ${request.url}`, redirect)
-          return createTabWithURL(redirect, browser)
+        // if subdomain request redirect to default public subdomain url
+        if (ipfsPathValidator.ipfsOrIpnsSubdomain(request.url)) {
+          redirectUrl = ipfsPathValidator.resolveToPublicSubdomainUrl(request.url, state.pubSubdomainGwURL)
+        } else {
+          redirectUrl = ipfsPathValidator.resolveToPublicUrl(request.url, state.pubGwURLString)
         }
+        log(`onCompleted: attempting to recover from HTTP Error ${request.statusCode} for ${request.url}`, redirectUrl)
+        return createTabWithURL({ redirectUrl }, browser)
       }
     }
   }
@@ -527,8 +536,8 @@ function isRecoverable (request, state, ipfsPathValidator) {
   return state.recoverFailedHttpRequests &&
     request.type === 'main_frame' &&
     (recoverableNetworkErrors.has(request.error) || recoverableHttpError(request.statusCode)) &&
-    ipfsPathValidator.publicIpfsOrIpnsResource(request.url) &&
-    !request.url.startsWith(state.pubGwURLString)
+    (ipfsPathValidator.publicIpfsOrIpnsResource(request.url) || ipfsPathValidator.ipfsOrIpnsSubdomain(request.url)) &&
+    !request.url.startsWith(state.pubGwURLString) && !request.url.includes(state.pubSubdomainGwURL.hostname)
 }
 
 // Recovery check for onErrorOccurred (request.error)

add-on/src/lib/options.js

@@ -11,6 +11,7 @@ exports.optionDefaults = Object.freeze({
   ipfsNodeType: buildDefaultIpfsNodeType(),
   ipfsNodeConfig: buildDefaultIpfsNodeConfig(),
   publicGatewayUrl: 'https://ipfs.io',
+  publicSubdomainGatewayUrl: 'https://dweb.link',
   useCustomGateway: true,
   noRedirectHostnames: [],
   automaticMode: true,

add-on/src/lib/state.js

@@ -17,6 +17,9 @@ function initState (options) {
   state.pubGwURL = safeURL(options.publicGatewayUrl)
   state.pubGwURLString = state.pubGwURL.toString()
   delete state.publicGatewayUrl
+  state.pubSubdomainGwURL = safeURL(options.publicSubdomainGatewayUrl)
+  state.pubSubdomainGwURLString = state.pubSubdomainGwURL.toString()
+  delete state.publicSubdomainGatewayUrl
   state.redirect = options.useCustomGateway
   delete state.useCustomGateway
   state.apiURL = safeURL(options.ipfsApiUrl)

add-on/src/options/forms/gateways-form.js

@@ -16,11 +16,13 @@ function gatewaysForm ({
   useCustomGateway,
   noRedirectHostnames,
   publicGatewayUrl,
+  publicSubdomainGatewayUrl,
   onOptionChange
 }) {
   const onCustomGatewayUrlChange = onOptionChange('customGatewayUrl', normalizeGatewayURL)
   const onUseCustomGatewayChange = onOptionChange('useCustomGateway')
   const onPublicGatewayUrlChange = onOptionChange('publicGatewayUrl', normalizeGatewayURL)
+  const onPublicSubdomainGatewayUrlChange = onOptionChange('publicSubdomainGatewayUrl', normalizeGatewayURL)
   const onNoRedirectHostnamesChange = onOptionChange('noRedirectHostnames', hostTextToArray)
   const mixedContentWarning = !secureContextUrl.test(customGatewayUrl)
   const supportRedirectToCustomGateway = ipfsNodeType !== 'embedded'
@@ -48,6 +50,29 @@ function gatewaysForm ({
               onchange=${onPublicGatewayUrlChange}
               value=${publicGatewayUrl} />
           </div>
+          <div>
+            <label for="publicSubdomainGatewayUrl">
+              <dl>
+                <dt>${browser.i18n.getMessage('option_publicSubdomainGatewayUrl_title')}</dt>
+                <dd>
+                  ${browser.i18n.getMessage('option_publicSubdomainGatewayUrl_description')}
+                  <p><a href="https://docs.ipfs.io/guides/guides/addressing/#subdomain-gateway" target="_blank">
+                    ${browser.i18n.getMessage('option_legend_readMore')}
+                  </a></p>
+                </dd>
+              </dl>
+            </label>
+            <input
+              id="publicSubdomainGatewayUrl"
+              type="url"
+              inputmode="url"
+              required
+              pattern="^https?://[^/]+/?$"
+              spellcheck="false"
+              title="Enter URL without any sub-path"
+              onchange=${onPublicSubdomainGatewayUrlChange}
+              value=${publicSubdomainGatewayUrl} />
+          </div>
           ${supportRedirectToCustomGateway && allowChangeOfCustomGateway ? html`
             <div>
               <label for="customGatewayUrl">

add-on/src/options/page.js

@@ -66,6 +66,7 @@ module.exports = function optionsPage (state, emit) {
     customGatewayUrl: state.options.customGatewayUrl,
     useCustomGateway: state.options.useCustomGateway,
     publicGatewayUrl: state.options.publicGatewayUrl,
+    publicSubdomainGatewayUrl: state.options.publicSubdomainGatewayUrl,
     noRedirectHostnames: state.options.noRedirectHostnames,
     onOptionChange
   })}

add-on/src/popup/browser-action/store.js

@@ -23,6 +23,7 @@ module.exports = (state, emitter) => {
     isIpfsOnline: false,
     ipfsApiUrl: null,
     publicGatewayUrl: null,
+    publicSubdomainGatewayUrl: null,
     gatewayAddress: null,
     swarmPeers: null,
     gatewayVersion: null,

test/functional/lib/ipfs-request-gateway-recover.test.js

@@ -46,6 +46,16 @@ describe('requestHandler.onCompleted:', function () { // HTTP-level errors
       state.recoverFailedHttpRequests = true
       state.dnslinkPolicy = false
     })
+    it('should do nothing if broken request is for the default subdomain gateway', async function () {
+      const request = urlRequestWithStatus('https://QmYzZgeWE7r8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h.ipfs.dweb.link/wiki/', 500)
+      await requestHandler.onCompleted(request)
+      assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called')
+    })
+    it('should redirect to default subdomain gateway on broken subdomain gateway request', async function () {
+      const request = urlRequestWithStatus('http://bafybeiemxf5abjwjbikoz4mc3a3dla6ual3jsgpdr4cjr3oz3evfyavhwq.ipfs.brokenexample.com/wiki/', 500)
+      await requestHandler.onCompleted(request)
+      assert.ok(browser.tabs.create.withArgs({ url: 'https://bafybeiemxf5abjwjbikoz4mc3a3dla6ual3jsgpdr4cjr3oz3evfyavhwq.ipfs.dweb.link/wiki/', active: true, openerTabId: 20 }).calledOnce, 'tabs.create should be called with default subdomain gateway URL')
+    })
     it('should do nothing if broken request is a non-IPFS request', async function () {
       const request = urlRequestWithStatus('https://wikipedia.org', 500)
       await requestHandler.onCompleted(request)
@@ -78,6 +88,11 @@ describe('requestHandler.onCompleted:', function () { // HTTP-level errors
       state.recoverFailedHttpRequests = false
       state.dnslinkPolicy = false
     })
+    it('should do nothing on failed subdomain gateway request', async function () {
+      const request = urlRequestWithStatus('https://QmYzZgeWE7r8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h.ipfs.brokendomain.com/wiki/', 500)
+      await requestHandler.onCompleted(request)
+      assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called')
+    })
     it('should do nothing on broken non-default public gateway IPFS request', async function () {
       const request = urlRequestWithStatus('https://nondefaultipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h', 500)
       await requestHandler.onCompleted(request)
@@ -120,6 +135,16 @@ describe('requestHandler.onErrorOccurred:', function () { // network errors
       state.recoverFailedHttpRequests = true
       state.dnslinkPolicy = false
     })
+    it('should do nothing if failed request is for the default subdomain gateway', async function () {
+      const request = urlRequestWithStatus('https://QmYzZgeWE7r8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h.ipfs.dweb.link/wiki/', 500)
+      await requestHandler.onErrorOccurred(request)
+      assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called')
+    })
+    it('should redirect to default subdomain gateway on failed subdomain gateway request', async function () {
+      const request = urlRequestWithStatus('http://bafybeiemxf5abjwjbikoz4mc3a3dla6ual3jsgpdr4cjr3oz3evfyavhwq.ipfs.brokenexample.com/wiki/', 500)
+      await requestHandler.onErrorOccurred(request)
+      assert.ok(browser.tabs.create.withArgs({ url: 'https://bafybeiemxf5abjwjbikoz4mc3a3dla6ual3jsgpdr4cjr3oz3evfyavhwq.ipfs.dweb.link/wiki/', active: true, openerTabId: 20 }).calledOnce, 'tabs.create should be called with default subdomain gateway URL')
+    })
     it('should do nothing if failed request is a non-IPFS request', async function () {
       const request = urlRequestWithNetworkError('https://wikipedia.org')
       await requestHandler.onErrorOccurred(request)
@@ -178,6 +203,11 @@ describe('requestHandler.onErrorOccurred:', function () { // network errors
       await requestHandler.onErrorOccurred(request)
       assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called')
     })
+    it('should do nothing on failed subdomain gateway request', async function () {
+      const request = urlRequestWithStatus('https://QmYzZgeWE7r8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h.ipfs.brokendomain.com/wiki/', 500)
+      await requestHandler.onErrorOccurred(request)
+      assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called')
+    })
     it('should do nothing on unreachable HTTP server with DNSLink', async function () {
       state.dnslinkPolicy = 'best-effort'
       dnslinkResolver.setDnslink('en.wikipedia-on-ipfs.org', '/ipfs/QmXoypizjW3WknFiJnKLwHCnL72vedxjQkDDP1mXWo6uco')