Skip to content

Commit ca26240

Browse files
lidellidel
authored
feat: recover from DNS failures (#797)
This change builds on top of #783 and: - adds support for recovery from DNS lookup failures - enables recovery for all HTTP Codes >= 400 - recovers `.eth` DNS failures bu reopening website on EthDNS gateway at `.eth.link` - simplifies some code paths and adds more tests Motivation: When a third-party IPFS gateway is discontinued or censored at DNS level, the IPFS Companion should retry request using currently active gateway set by the user (public or local). We also want to recover in situation when website with DNSLink has a valid DNS `TXT` record, but HTTP behind `A` record is down or unreachable. *Note*: right now the only real use for DNS recovery is support of .eth TLD via .eth.link gateway, however in the future this could provide means of working around DNS-based censorship (eg. by executing DNSLink lookups over libp2p as a fallback). closes #678, closes #640
1 parent 614da95 commit ca26240

File tree

4 files changed

+135
-95
lines changed

4 files changed

+135
-95
lines changed

‎add-on/src/lib/dnslink.js

+1-2
Original file line numberDiff line numberDiff line change
@@ -180,8 +180,7 @@ module.exports = function createDnslinkResolver (getState) {
180180
if (typeof url === 'string') {
181181
url = new URL(url)
182182
}
183-
const fqdn = url.hostname
184-
return `/ipns/${fqdn}${url.pathname}${url.search}${url.hash}`
183+
return `/ipns/${url.hostname}${url.pathname}${url.search}${url.hash}`
185184
},
186185

187186
// Test if URL contains a valid DNSLink FQDN

‎add-on/src/lib/ipfs-request.js

+68-74
Original file line numberDiff line numberDiff line change
@@ -9,36 +9,20 @@ const LRU = require('lru-cache')
99
const IsIpfs = require('is-ipfs')
1010
const isFQDN = require('is-fqdn')
1111
const { pathAtHttpGateway } = require('./ipfs-path')
12+
1213
const redirectOptOutHint = 'x-ipfs-companion-no-redirect'
13-
const recoverableErrors = new Set([
14+
const recoverableNetworkErrors = new Set([
1415
// Firefox
16+
'NS_ERROR_UNKNOWN_HOST', // dns failure
1517
'NS_ERROR_NET_TIMEOUT', // eg. httpd is offline
1618
'NS_ERROR_NET_RESET', // failed to load because the server kept reseting the connection
1719
'NS_ERROR_NET_ON_RESOLVED', // no network
1820
// Chrome
21+
'net::ERR_NAME_NOT_RESOLVED', // dns failure
1922
'net::ERR_CONNECTION_TIMED_OUT', // eg. httpd is offline
2023
'net::ERR_INTERNET_DISCONNECTED' // no network
2124
])
22-
23-
const recoverableErrorCodes = new Set([
24-
404,
25-
408,
26-
410,
27-
415,
28-
451,
29-
500,
30-
502,
31-
503,
32-
504,
33-
509,
34-
520,
35-
521,
36-
522,
37-
523,
38-
524,
39-
525,
40-
526
41-
])
25+
const recoverableHttpError = (code) => code && code >= 400
4226

4327
// Request modifier provides event listeners for the various stages of making an HTTP request
4428
// API Details: https://developer.mozilla.org/en-US/Add-ons/WebExtensions/API/webRequest
@@ -171,11 +155,7 @@ function createRequestModifier (getState, dnslinkResolver, ipfsPathValidator, ru
171155
// This is a good place to listen if you want to modify HTTP request headers.
172156
onBeforeSendHeaders (request) {
173157
const state = getState()
174-
175-
// Skip if IPFS integrations are inactive
176-
if (!state.active) {
177-
return
178-
}
158+
if (!state.active) return
179159

180160
// Special handling of requests made to API
181161
if (request.url.startsWith(state.apiURLString)) {
@@ -286,11 +266,7 @@ function createRequestModifier (getState, dnslinkResolver, ipfsPathValidator, ru
286266
// You can use this event to modify HTTP response headers or do a very late redirect.
287267
onHeadersReceived (request) {
288268
const state = getState()
289-
290-
// Skip if IPFS integrations are inactive
291-
if (!state.active) {
292-
return
293-
}
269+
if (!state.active) return
294270

295271
// Special handling of requests made to API
296272
if (request.url.startsWith(state.apiURLString)) {
@@ -387,58 +363,53 @@ function createRequestModifier (getState, dnslinkResolver, ipfsPathValidator, ru
387363
},
388364

389365
// browser.webRequest.onErrorOccurred
390-
// Fired when a request could not be processed due to an error:
391-
// for example, a lack of Internet connectivity.
366+
// Fired when a request could not be processed due to an error on network level.
367+
// For example: TCP timeout, DNS lookup failure
392368
async onErrorOccurred (request) {
393369
const state = getState()
394-
395-
// Skip if IPFS integrations are inactive or request is marked as ignored
396-
if (!state.active || isIgnored(request.requestId)) {
397-
return
370+
if (!state.active) return
371+
372+
// Check if error can be recovered via EthDNS
373+
if (isRecoverableViaEthDNS(request, state)) {
374+
const url = new URL(request.url)
375+
url.hostname = `${url.hostname}.link`
376+
const redirect = { redirectUrl: url.toString() }
377+
log(`onErrorOccurred: attempting to recover from DNS error (${request.error}) using EthDNS for ${request.url}`, redirect.redirectUrl)
378+
return createTabWithURL(redirect, browser)
398379
}
399380

400-
// console.log('onErrorOccurred:' + request.error)
401-
// console.log('onErrorOccurred', request)
402-
// Check if error is final and can be recovered via DNSLink
403-
let redirect
404-
const recoverableViaDnslink =
405-
state.dnslinkPolicy &&
406-
request.type === 'main_frame' &&
407-
recoverableErrors.has(request.error)
408-
if (recoverableViaDnslink && dnslinkResolver.canLookupURL(request.url)) {
409-
// Explicit call to ignore global DNSLink policy and force DNS TXT lookup
410-
const cachedDnslink = dnslinkResolver.readAndCacheDnslink(new URL(request.url).hostname)
411-
redirect = dnslinkResolver.dnslinkRedirect(request.url, cachedDnslink)
412-
log(`onErrorOccurred: attempting to recover using dnslink for ${request.url}`, redirect)
381+
// Check if error can be recovered via DNSLink
382+
if (isRecoverableViaDNSLink(request, state, dnslinkResolver)) {
383+
const { hostname } = new URL(request.url)
384+
const dnslink = dnslinkResolver.readAndCacheDnslink(hostname)
385+
if (dnslink) {
386+
const redirect = dnslinkResolver.dnslinkRedirect(request.url, dnslink)
387+
log(`onErrorOccurred: attempting to recover from network error (${request.error}) using dnslink for ${request.url}`, redirect.redirectUrl)
388+
return createTabWithURL(redirect, browser)
389+
}
413390
}
414-
// if error cannot be recovered via DNSLink
415-
// direct the request to the public gateway
416-
const recoverable = isRecoverable(request, state, ipfsPathValidator)
417-
if (!redirect && recoverable) {
391+
392+
// Check if error can be recovered by opening same content-addresed path
393+
// using active gateway (public or local, depending on redirect state)
394+
if (isRecoverable(request, state, ipfsPathValidator)) {
418395
const redirectUrl = ipfsPathValidator.resolveToPublicUrl(request.url, state.pubGwURLString)
419-
redirect = { redirectUrl }
420-
log(`onErrorOccurred: attempting to recover failed request for ${request.url}`, redirect)
421-
}
422-
// We can't redirect in onErrorOccurred, so if DNSLink is present
423-
// recover by opening IPNS version in a new tab
424-
// TODO: add tests and demo
425-
if (redirect) {
426-
createTabWithURL(redirect, browser)
396+
log(`onErrorOccurred: attempting to recover from network error (${request.error}) for ${request.url}`, redirectUrl)
397+
return createTabWithURL({ redirectUrl }, browser)
427398
}
428399
},
429400

401+
// browser.webRequest.onCompleted
402+
// Fired when HTTP request is completed (successfully or with an error code)
430403
async onCompleted (request) {
431404
const state = getState()
432-
433-
const recoverable =
434-
isRecoverable(request, state, ipfsPathValidator) &&
435-
recoverableErrorCodes.has(request.statusCode)
436-
if (recoverable) {
405+
if (!state.active) return
406+
if (request.statusCode === 200) return // finish if no error to recover from
407+
if (isRecoverable(request, state, ipfsPathValidator)) {
437408
const redirectUrl = ipfsPathValidator.resolveToPublicUrl(request.url, state.pubGwURLString)
438409
const redirect = { redirectUrl }
439410
if (redirect) {
440-
log(`onCompleted: attempting to recover failed request for ${request.url}`, redirect)
441-
createTabWithURL(redirect, browser)
411+
log(`onCompleted: attempting to recover from HTTP Error ${request.statusCode} for ${request.url}`, redirect)
412+
return createTabWithURL(redirect, browser)
442413
}
443414
}
444415
}
@@ -548,18 +519,41 @@ function findHeaderIndex (name, headers) {
548519
return headers.findIndex(x => x.name && x.name.toLowerCase() === name.toLowerCase())
549520
}
550521

551-
// utility functions for handling redirects
552-
// from onErrorOccurred and onCompleted
522+
// RECOVERY OF FAILED REQUESTS
523+
// ===================================================================
524+
525+
// Recovery check for onErrorOccurred (request.error) and onCompleted (request.statusCode)
553526
function isRecoverable (request, state, ipfsPathValidator) {
554527
return state.recoverFailedHttpRequests &&
528+
request.type === 'main_frame' &&
529+
(recoverableNetworkErrors.has(request.error) || recoverableHttpError(request.statusCode)) &&
555530
ipfsPathValidator.publicIpfsOrIpnsResource(request.url) &&
556-
!request.url.startsWith(state.pubGwURLString) &&
557-
request.type === 'main_frame'
531+
!request.url.startsWith(state.pubGwURLString)
532+
}
533+
534+
// Recovery check for onErrorOccurred (request.error)
535+
function isRecoverableViaDNSLink (request, state, dnslinkResolver) {
536+
const recoverableViaDnslink =
537+
state.recoverFailedHttpRequests &&
538+
request.type === 'main_frame' &&
539+
state.dnslinkPolicy &&
540+
recoverableNetworkErrors.has(request.error)
541+
return recoverableViaDnslink && dnslinkResolver.canLookupURL(request.url)
542+
}
543+
544+
// Recovery check for onErrorOccurred (request.error)
545+
function isRecoverableViaEthDNS (request, state) {
546+
return state.recoverFailedHttpRequests &&
547+
request.type === 'main_frame' &&
548+
recoverableNetworkErrors.has(request.error) &&
549+
new URL(request.url).hostname.endsWith('.eth')
558550
}
559551

552+
// We can't redirect in onErrorOccurred/onCompleted
553+
// Indead, we recover by opening URL in a new tab that replaces the failed one
560554
async function createTabWithURL (redirect, browser) {
561555
const currentTabId = await browser.tabs.query({ active: true, currentWindow: true }).then(tabs => tabs[0].id)
562-
await browser.tabs.create({
556+
return browser.tabs.create({
563557
active: true,
564558
openerTabId: currentTabId,
565559
url: redirect.redirectUrl

‎add-on/src/options/forms/experiments-form.js

+9-9
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,15 @@ function experimentsForm ({
5959
</label>
6060
<div>${switchToggle({ id: 'catchUnhandledProtocols', checked: catchUnhandledProtocols, onchange: onCatchUnhandledProtocolsChange })}</div>
6161
</div>
62+
<div>
63+
<label for="recoverFailedHttpRequests">
64+
<dl>
65+
<dt>${browser.i18n.getMessage('option_recoverFailedHttpRequests_title')}</dt>
66+
<dd>${browser.i18n.getMessage('option_recoverFailedHttpRequests_description')}</dd>
67+
</dl>
68+
</label>
69+
<div>${switchToggle({ id: 'recoverFailedHttpRequests', checked: recoverFailedHttpRequests, onchange: onrecoverFailedHttpRequestsChange })}</div>
70+
</div>
6271
<div>
6372
<label for="linkify">
6473
<dl>
@@ -98,15 +107,6 @@ function experimentsForm ({
98107
</option>
99108
</select>
100109
</div>
101-
<div>
102-
<label for="recoverFailedHttpRequests">
103-
<dl>
104-
<dt>${browser.i18n.getMessage('option_recoverFailedHttpRequests_title')}</dt>
105-
<dd>${browser.i18n.getMessage('option_recoverFailedHttpRequests_description')}</dd>
106-
</dl>
107-
</label>
108-
<div>${switchToggle({ id: 'recoverFailedHttpRequests', checked: recoverFailedHttpRequests, onchange: onrecoverFailedHttpRequestsChange })}</div>
109-
</div>
110110
<div>
111111
<label for="detectIpfsPathHeader">
112112
<dl>

‎test/functional/lib/ipfs-request-gateway-recover.test.js

+57-10
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,11 @@ const urlRequestWithStatus = (url, statusCode = 200, type = 'main_frame') => {
1818
return { ...url2request(url, type), statusCode }
1919
}
2020

21-
describe('requestHandler.onCompleted:', function () {
21+
const urlRequestWithNetworkError = (url, error = 'net::ERR_CONNECTION_TIMED_OUT', type = 'main_frame') => {
22+
return { ...url2request(url, type), error }
23+
}
24+
25+
describe('requestHandler.onCompleted:', function () { // HTTP-level errors
2226
let state, dnslinkResolver, ipfsPathValidator, requestHandler, runtime
2327

2428
before(function () {
@@ -40,6 +44,7 @@ describe('requestHandler.onCompleted:', function () {
4044
describe('with recoverFailedHttpRequests=true', function () {
4145
beforeEach(function () {
4246
state.recoverFailedHttpRequests = true
47+
state.dnslinkPolicy = false
4348
})
4449
it('should do nothing if broken request is a non-IPFS request', async function () {
4550
const request = urlRequestWithStatus('https://wikipedia.org', 500)
@@ -61,7 +66,7 @@ describe('requestHandler.onCompleted:', function () {
6166
await requestHandler.onCompleted(request)
6267
assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called')
6368
})
64-
it('should redirect broken non-default public gateway IPFS request to public gateway', async function () {
69+
it('should recover from unreachable third party public gateway by reopening on the public gateway', async function () {
6570
const request = urlRequestWithStatus('https://nondefaultipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h', 500)
6671
await requestHandler.onCompleted(request)
6772
assert.ok(browser.tabs.create.withArgs({ url: 'https://ipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h', active: true, openerTabId: 20 }).calledOnce, 'tabs.create should be called with IPFS default public gateway URL')
@@ -71,6 +76,7 @@ describe('requestHandler.onCompleted:', function () {
7176
describe('with recoverFailedHttpRequests=false', function () {
7277
beforeEach(function () {
7378
state.recoverFailedHttpRequests = false
79+
state.dnslinkPolicy = false
7480
})
7581
it('should do nothing on broken non-default public gateway IPFS request', async function () {
7682
const request = urlRequestWithStatus('https://nondefaultipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h', 500)
@@ -90,7 +96,7 @@ describe('requestHandler.onCompleted:', function () {
9096
})
9197
})
9298

93-
describe('requestHandler.onErrorOccurred:', function () {
99+
describe('requestHandler.onErrorOccurred:', function () { // network errors
94100
let state, dnslinkResolver, ipfsPathValidator, requestHandler, runtime
95101

96102
before(function () {
@@ -112,42 +118,83 @@ describe('requestHandler.onErrorOccurred:', function () {
112118
describe('with recoverFailedHttpRequests=true', function () {
113119
beforeEach(function () {
114120
state.recoverFailedHttpRequests = true
121+
state.dnslinkPolicy = false
115122
})
116123
it('should do nothing if failed request is a non-IPFS request', async function () {
117-
const request = url2request('https://wikipedia.org', 500)
124+
const request = urlRequestWithNetworkError('https://wikipedia.org')
118125
await requestHandler.onErrorOccurred(request)
119126
assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called')
120127
})
121128
it('should do nothing if failed request is a non-public IPFS request', async function () {
122-
const request = url2request('http://127.0.0.1:8080/ipfs/QmYzZgeWE7r8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h', 500)
129+
const request = urlRequestWithNetworkError('http://127.0.0.1:8080/ipfs/QmYzZgeWE7r8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h')
123130
await requestHandler.onErrorOccurred(request)
124131
assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called')
125132
})
126133
it('should do nothing if failed request is to the default public gateway', async function () {
127-
const request = url2request('https://ipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h', 500)
134+
const request = urlRequestWithNetworkError('https://ipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h')
128135
await requestHandler.onErrorOccurred(request)
129136
assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called')
130137
})
131138
it('should do nothing if failed request is not a \'main_frame\' request', async function () {
132-
const request = url2request('https://nondefaultipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h', 'stylesheet')
139+
const requestType = 'stylesheet'
140+
const request = urlRequestWithNetworkError('https://nondefaultipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h', 'net::ERR_NAME_NOT_RESOLVED', requestType)
133141
await requestHandler.onErrorOccurred(request)
134142
assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called')
135143
})
136-
it('should redirect failed non-default public gateway IPFS request to public gateway', async function () {
137-
const request = url2request('https://nondefaultipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h')
144+
it('should recover from unreachable third party public gateway by reopening on the public gateway', async function () {
145+
const request = urlRequestWithNetworkError('https://nondefaultipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h')
138146
await requestHandler.onErrorOccurred(request)
139147
assert.ok(browser.tabs.create.withArgs({ url: 'https://ipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h', active: true, openerTabId: 20 }).calledOnce, 'tabs.create should be called with IPFS default public gateway URL')
140148
})
149+
it('should recover from unreachable HTTP server by reopening DNSLink on the public gateway', async function () {
150+
state.dnslinkPolicy = 'best-effort'
151+
dnslinkResolver.setDnslink('en.wikipedia-on-ipfs.org', '/ipfs/QmXoypizjW3WknFiJnKLwHCnL72vedxjQkDDP1mXWo6uco')
152+
const expectedUrl = 'http://127.0.0.1:8080/ipns/en.wikipedia-on-ipfs.org/'
153+
const request = urlRequestWithNetworkError('https://en.wikipedia-on-ipfs.org/')
154+
await requestHandler.onErrorOccurred(request)
155+
assert.ok(browser.tabs.create.withArgs({ url: expectedUrl, active: true, openerTabId: 20 }).calledOnce, 'tabs.create should be called with ENS resource on local gateway URL')
156+
dnslinkResolver.clearCache()
157+
})
158+
it('should recover from failed DNS for .eth opening it on EthDNS gateway at .eth.link', async function () {
159+
state.dnslinkPolicy = 'best-effort'
160+
dnslinkResolver.setDnslink('almonit.eth', false)
161+
dnslinkResolver.setDnslink('almonit.eth.link', '/ipfs/QmPH7VMnfFKvrr7kLXNRwuxjYRLWnfcxPvnWs8ipyWAQK2')
162+
const dnsFailure = 'net::ERR_NAME_NOT_RESOLVED' // chrome code
163+
const expectedUrl = 'https://almonit.eth.link/'
164+
const request = urlRequestWithNetworkError('https://almonit.eth', dnsFailure)
165+
await requestHandler.onErrorOccurred(request)
166+
assert.ok(browser.tabs.create.withArgs({ url: expectedUrl, active: true, openerTabId: 20 }).calledOnce, 'tabs.create should be called with ENS resource on local gateway URL')
167+
dnslinkResolver.clearCache()
168+
})
141169
})
142170

143171
describe('with recoverFailedHttpRequests=false', function () {
144172
beforeEach(function () {
145173
state.recoverFailedHttpRequests = false
174+
state.dnslinkPolicy = false
175+
})
176+
it('should do nothing on unreachable third party public gateway', async function () {
177+
const request = urlRequestWithNetworkError('https://nondefaultipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h')
178+
await requestHandler.onErrorOccurred(request)
179+
assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called')
180+
})
181+
it('should do nothing on unreachable HTTP server with DNSLink', async function () {
182+
state.dnslinkPolicy = 'best-effort'
183+
dnslinkResolver.setDnslink('en.wikipedia-on-ipfs.org', '/ipfs/QmXoypizjW3WknFiJnKLwHCnL72vedxjQkDDP1mXWo6uco')
184+
const request = urlRequestWithNetworkError('https://en.wikipedia-on-ipfs.org')
185+
await requestHandler.onErrorOccurred(request)
186+
assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called')
187+
dnslinkResolver.clearCache()
146188
})
147189
it('should do nothing on failed non-default public gateway IPFS request', async function () {
148-
const request = url2request('https://nondefaultipfs.io/ipfs/QmYbZgeWE7y8HXkH8zbb8J9ddHQvp8LTqm6isL791eo14h')
190+
state.dnslinkPolicy = 'best-effort'
191+
dnslinkResolver.setDnslink('almonit.eth', false)
192+
dnslinkResolver.setDnslink('almonit.eth.link', '/ipfs/QmPH7VMnfFKvrr7kLXNRwuxjYRLWnfcxPvnWs8ipyWAQK2')
193+
const dnsFailure = 'net::ERR_NAME_NOT_RESOLVED' // chrome code
194+
const request = urlRequestWithNetworkError('https://almonit.eth', dnsFailure)
149195
await requestHandler.onErrorOccurred(request)
150196
assert.ok(browser.tabs.create.notCalled, 'tabs.create should not be called')
197+
dnslinkResolver.clearCache()
151198
})
152199
})
153200

0 commit comments

Comments
 (0)