wip: js-ipfs in Brave

lidel · lidel · commit 4dda1b1d7a08 · 2020-08-18T15:50:14.000+02:00
- the latest js-ipfs builds and runs in the browser
- 4x delegates and preloads enabled
- 2x webrtc star provides signaling

Known Issues:

- the node starts, but struggles to find content, and there is a separate
  issue of infinite restart when user enables and disables it too fast
- import via quick import page fails due to API discrepancy
diff --git a/add-on/src/lib/ipfs-client/embedded-chromesockets/config.js b/add-on/src/lib/ipfs-client/embedded-chromesockets/config.js
@@ -30,14 +30,17 @@ const chromeDefaultOpts = {
          but Web UI needs API (can't use window.ipfs due to sandboxing)
       */
       Swarm: [
-        // optional ws-star signaling provides a backup for non-LAN peer discovery
+        // additional signaling service provides a backup for non-LAN peer discovery
         // (this will be removed when autorelay and DHT are stable in js-ipfs)
-        '/dns4/wrtc-star.discovery.libp2p.io/tcp/443/wss/p2p-webrtc-star'
+        '/dns4/wrtc-star1.par.dwebops.pub/tcp/443/wss/p2p-webrtc-star',
+        '/dns4/wrtc-star2.sjc.dwebops.pub/tcp/443/wss/p2p-webrtc-star'
       ],
       // Delegated Content and Peer Routing: https://github.com/ipfs/js-ipfs/pull/2195
       Delegates: [
+        '/dns4/node0.delegate.ipfs.io/tcp/443/https',
         '/dns4/node1.delegate.ipfs.io/tcp/443/https',
-        '/dns4/node0.delegate.ipfs.io/tcp/443/https'
+        '/dns4/node2.delegate.ipfs.io/tcp/443/https',
+        '/dns4/node3.delegate.ipfs.io/tcp/443/https'
       ]
     },
     Discovery: {
@@ -49,47 +52,51 @@ const chromeDefaultOpts = {
     Swarm: {
       ConnMgr: {
         LowWater: 50,
-        HighWater: 150
+        HighWater: 200
       }
     },
     Bootstrap: [
-      // Prioritize TCP Bootstrappers from https://github.com/ipfs/js-ipfs/blob/v0.37.1/src/core/runtime/config-nodejs.js#L22
-      '/ip4/104.236.176.52/tcp/4001/ipfs/QmSoLnSGccFuZQJzRadHn95W2CrSFmZuTdDWP8HXaHca9z',
-      '/ip4/104.131.131.82/tcp/4001/ipfs/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ',
-      '/ip4/104.236.179.241/tcp/4001/ipfs/QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM',
-      '/ip4/162.243.248.213/tcp/4001/ipfs/QmSoLueR4xBeUbY9WZ9xGUUxunbKWcrNFTDAadQJmocnWm',
-      '/ip4/128.199.219.111/tcp/4001/ipfs/QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu',
-      '/ip4/104.236.76.40/tcp/4001/ipfs/QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64',
-      '/ip4/178.62.158.247/tcp/4001/ipfs/QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd',
-      '/ip4/178.62.61.185/tcp/4001/ipfs/QmSoLMeWqB7YGVLJN3pNLQpmmEk35v6wYtsMGLzSr5QBU3',
-      '/ip4/104.236.151.122/tcp/4001/ipfs/QmSoLju6m7xTh3DuokvT3886QRYqxAzb1kShaanJgW36yx',
-      '/ip6/2604:a880:1:20::1f9:9001/tcp/4001/ipfs/QmSoLnSGccFuZQJzRadHn95W2CrSFmZuTdDWP8HXaHca9z',
-      '/ip6/2604:a880:1:20::203:d001/tcp/4001/ipfs/QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM',
-      '/ip6/2604:a880:0:1010::23:d001/tcp/4001/ipfs/QmSoLueR4xBeUbY9WZ9xGUUxunbKWcrNFTDAadQJmocnWm',
-      '/ip6/2400:6180:0:d0::151:6001/tcp/4001/ipfs/QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu',
-      '/ip6/2604:a880:800:10::4a:5001/tcp/4001/ipfs/QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64',
-      '/ip6/2a03:b0c0:0:1010::23:1001/tcp/4001/ipfs/QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd',
-      '/ip6/2a03:b0c0:1:d0::e7:1/tcp/4001/ipfs/QmSoLMeWqB7YGVLJN3pNLQpmmEk35v6wYtsMGLzSr5QBU3',
-      '/ip6/2604:a880:1:20::1d9:6001/tcp/4001/ipfs/QmSoLju6m7xTh3DuokvT3886QRYqxAzb1kShaanJgW36yx',
+      // Prioritize TCP Bootstrappers from https://github.com/ipfs/js-ipfs/blob/ipfs%400.49.0/packages/ipfs/src/core/runtime/config-nodejs.js
+      '/ip4/104.236.176.52/tcp/4001/p2p/QmSoLnSGccFuZQJzRadHn95W2CrSFmZuTdDWP8HXaHca9z',
+      '/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ',
+      '/ip4/104.236.179.241/tcp/4001/p2p/QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM',
+      '/ip4/162.243.248.213/tcp/4001/p2p/QmSoLueR4xBeUbY9WZ9xGUUxunbKWcrNFTDAadQJmocnWm',
+      '/ip4/128.199.219.111/tcp/4001/p2p/QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu',
+      '/ip4/104.236.76.40/tcp/4001/p2p/QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64',
+      '/ip4/178.62.158.247/tcp/4001/p2p/QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd',
+      '/ip4/178.62.61.185/tcp/4001/p2p/QmSoLMeWqB7YGVLJN3pNLQpmmEk35v6wYtsMGLzSr5QBU3',
+      '/ip4/104.236.151.122/tcp/4001/p2p/QmSoLju6m7xTh3DuokvT3886QRYqxAzb1kShaanJgW36yx',
+      '/ip6/2604:a880:1:20::1f9:9001/tcp/4001/p2p/QmSoLnSGccFuZQJzRadHn95W2CrSFmZuTdDWP8HXaHca9z',
+      '/ip6/2604:a880:1:20::203:d001/tcp/4001/p2p/QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM',
+      '/ip6/2604:a880:0:1010::23:d001/tcp/4001/p2p/QmSoLueR4xBeUbY9WZ9xGUUxunbKWcrNFTDAadQJmocnWm',
+      '/ip6/2400:6180:0:d0::151:6001/tcp/4001/p2p/QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu',
+      '/ip6/2604:a880:800:10::4a:5001/tcp/4001/p2p/QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64',
+      '/ip6/2a03:b0c0:0:1010::23:1001/tcp/4001/p2p/QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd',
+      '/ip6/2a03:b0c0:1:d0::e7:1/tcp/4001/p2p/QmSoLMeWqB7YGVLJN3pNLQpmmEk35v6wYtsMGLzSr5QBU3',
+      '/ip6/2604:a880:1:20::1d9:6001/tcp/4001/p2p/QmSoLju6m7xTh3DuokvT3886QRYqxAzb1kShaanJgW36yx',
       // Twist: connect to preload nodes, but over tcp :)
-      '/dns4/node0.preload.ipfs.io/tcp/4001/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic',
-      '/dns4/node1.preload.ipfs.io/tcp/4001/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6',
-      // WebSockets versions from https://github.com/ipfs/js-ipfs/blob/v0.37.1/src/core/runtime/config-browser.js#L20
+      '/dns4/node0.preload.ipfs.io/tcp/443/wss/p2p/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic',
+      '/dns4/node1.preload.ipfs.io/tcp/443/wss/p2p/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6',
+      '/dns4/node2.preload.ipfs.io/tcp/443/wss/p2p/QmV7gnbW5VTcJ3oyM2Xk1rdFBJ3kTkvxc87UFGsun29STS',
+      '/dns4/node3.preload.ipfs.io/tcp/443/wss/p2p/QmY7JB6MQXhxHvq7dBDh4HpbH29v4yE9JRadAVpndvzySN',
+      // WebSockets versions from https://github.com/ipfs/js-ipfs/blob/ipfs%400.49.0/packages/ipfs/src/core/runtime/config-browser.js
       '/dns4/ams-1.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd',
       '/dns4/lon-1.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmSoLMeWqB7YGVLJN3pNLQpmmEk35v6wYtsMGLzSr5QBU3',
       '/dns4/sfo-3.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM',
       '/dns4/sgp-1.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu',
       '/dns4/nyc-1.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmSoLueR4xBeUbY9WZ9xGUUxunbKWcrNFTDAadQJmocnWm',
       '/dns4/nyc-2.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64',
-      '/dns4/node0.preload.ipfs.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic',
-      '/dns4/node1.preload.ipfs.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6'
+      '/dns4/node0.preload.ipfs.io/tcp/443/wss/p2p/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic',
+      '/dns4/node1.preload.ipfs.io/tcp/443/wss/p2p/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6',
+      '/dns4/node2.preload.ipfs.io/tcp/443/wss/p2p/QmV7gnbW5VTcJ3oyM2Xk1rdFBJ3kTkvxc87UFGsun29STS',
+      '/dns4/node3.preload.ipfs.io/tcp/443/wss/p2p/QmY7JB6MQXhxHvq7dBDh4HpbH29v4yE9JRadAVpndvzySN'
     ]
   },
-  // preload: { enabled: false, addresses: [] }
-  /* TODO: when we have p2p transport, are preloads still needed? should Brave have own nodes? */
   preload: {
     enabled: true,
     addresses: [
+      '/dns4/node3.preload.ipfs.io/tcp/443/https',
+      '/dns4/node2.preload.ipfs.io/tcp/443/https',
       '/dns4/node1.preload.ipfs.io/tcp/443/https',
       '/dns4/node0.preload.ipfs.io/tcp/443/https'
     ]
@@ -140,11 +147,29 @@ async function buildConfig (opts, log) {
           },
           webRTCStar: {
             enabled: true
+          },
+          dht: {
+            kBucketSize: 20,
+            enabled: false,
+            clientMode: false, /* TODO experiment with true */
+            randomWalk: {
+              enabled: false
+            }
+          },
+          pubsub: {
+            enabled: true,
+            emitSelf: true
           }
         }
       })
 
-      libp2pOptions.metrics = { enabled: false }
+      /* TODO: find out the best valus for chrome apps context
+      libp2pOptions.dialer = {
+        // https://github.com/ipfs/js-ipfs/blob/ipfs%400.49.0/packages/ipfs/src/core/runtime/libp2p-browser.js#L14
+        maxParallelDials: 150, // 150 total parallel multiaddr dials
+        maxDialsPerPeer: 4, // Allow 4 multiaddrs to be dialed per peer in parallel
+        dialTimeout: 10e3 // 10 second dial timeout per peer dial
+      }*/
 
       log('initializing libp2p with libp2pOptions', libp2pOptions)
       return new Libp2p(libp2pOptions)
@@ -170,11 +195,11 @@ async function buildConfig (opts, log) {
   return ipfsNodeConfig
 }
 
-async function syncConfig (ipfs, opts, log) {
+async function syncConfig (liveConfig, log) {
   const storedConfig = await browser.storage.local.get('ipfsNodeConfig')
   if (storedConfig && storedConfig.ipfsNodeConfig) {
-    const maGw = await ipfs.config.get('Addresses.Gateway')
-    const maApi = await ipfs.config.get('Addresses.API')
+    const maGw = liveConfig.config.Addresses.Gateway
+    const maApi = liveConfig.config.Addresses.API
     const httpGw = multiaddr2httpUrl(maGw)
     const httpApi = multiaddr2httpUrl(maApi)
     // update config in browser.storage to ports from js-ipfs instance
@@ -191,7 +216,7 @@ async function syncConfig (ipfs, opts, log) {
       changes.ipfsNodeConfig = JSON.stringify(cfg, null, 2)
     }
     // update runtime config in place
-    Object.assign(opts, changes)
+    Object.assign(liveConfig, changes)
     // save config to browser.storage (triggers async client restart if ports changed)
     log(`synchronizing ipfsNodeConfig with customGatewayUrl (${changes.customGatewayUrl}) and ipfsApiUrl (${changes.ipfsApiUrl})`)
     await browser.storage.local.set(changes)
diff --git a/add-on/src/lib/ipfs-client/embedded-chromesockets/index.js b/add-on/src/lib/ipfs-client/embedded-chromesockets/index.js
@@ -30,11 +30,14 @@ exports.init = async function init (opts) {
 
   log('creating js-ipfs with opts: ', ipfsOpts)
   node = await Ipfs.create(ipfsOpts)
+  await node.start()
 
   log('starting HTTP servers with opts: ', ipfsOpts)
   const httpServers = new HttpApi(node, ipfsOpts)
   nodeHttpApi = await httpServers.start()
-  await syncConfig(node, opts, log)
+
+  await syncConfig(ipfsOpts, log)
+
   return node
 }
 
diff --git a/add-on/src/lib/ipfs-client/index.js b/add-on/src/lib/ipfs-client/index.js
@@ -7,44 +7,59 @@ const log = debug('ipfs-companion:client')
 log.error = debug('ipfs-companion:client:error')
 
 const browser = require('webextension-polyfill')
+const mortice = require('mortice')
 const external = require('./external')
 const embedded = require('./embedded')
 const embeddedWithChromeSockets = require('./embedded-chromesockets')
 const { precache } = require('../precache')
 
+// ensure single client at all times, and no overlap between init and destroy
+// TODO: replace mortice with state machine?
+const mutex = mortice('ipfs-client-lock')
 let client
 
 async function initIpfsClient (opts) {
-  log('init ipfs client')
   await destroyIpfsClient()
-  switch (opts.ipfsNodeType) {
-    case 'embedded':
-      client = embedded
-      break
-    case 'embedded:chromesockets':
-      client = embeddedWithChromeSockets
-      break
-    case 'external':
-      client = external
-      break
-    default:
-      throw new Error(`Unsupported ipfsNodeType: ${opts.ipfsNodeType}`)
-  }
+  const release = await mutex.writeLock()
+  try {
+    log('init ipfs client')
+    switch (opts.ipfsNodeType) {
+      case 'embedded':
+        client = embedded
+        break
+      case 'embedded:chromesockets':
+        client = embeddedWithChromeSockets
+        break
+      case 'external':
+        client = external
+        break
+      default:
+        throw new Error(`Unsupported ipfsNodeType: ${opts.ipfsNodeType}`)
+    }
 
-  const instance = await client.init(opts)
-  easeApiChanges(instance)
-  _reloadIpfsClientDependents(instance) // async (API is present)
-  return instance
+    const instance = await client.init(opts)
+    easeApiChanges(instance)
+    _reloadIpfsClientDependents(instance) // async (API is present)
+    return instance
+  } finally {
+    release()
+  }
 }
 
 async function destroyIpfsClient () {
-  if (client && client.destroy) {
-    try {
-      await client.destroy()
-    } finally {
-      client = null
-      await _reloadIpfsClientDependents() // sync (API stopped working)
+  const release = await mutex.writeLock()
+  log('destroy ipfs client')
+  try {
+    if (client && client.destroy) {
+      try {
+        await client.destroy()
+      } finally {
+        client = null
+        await _reloadIpfsClientDependents() // sync (API stopped working)
+      }
     }
+  } finally {
+    release()
   }
 }
 
diff --git a/add-on/src/popup/browser-action/ipfs-version.js b/add-on/src/popup/browser-action/ipfs-version.js
@@ -11,7 +11,7 @@ function statusEntry ({ label, labelLegend, title, value, check, valueClass = ''
   value = value || value === 0 ? value : offline
   return html`
       <div title="${labelLegend}" class="ma0 pa0" style="line-height: 0.25">
-        <span class="f7 tr monospace force-select-all ${valueClass}" title="${title}">${value}</span>
+        <span class="f7 tr monospace force-select-all ${valueClass}" title="${title}">${value.substring(0, 13)}</span>
       </div>
     `
 }
diff --git a/package.json b/package.json
@@ -75,7 +75,7 @@
     "iso-stream-http": "0.1.2",
     "stream-http": "npm:iso-stream-http@0.1.2",
     "stream": "npm:readable-stream@3.6.0",
-    "prom-client": "npm:empty-module@0.0.2",
+    "prom-client": "file:./scripts/prom-client-stub",
     "prometheus-gc-stats": "npm:empty-module@0.0.2",
     "pull-to-stream": "0.1.1"
   },
@@ -147,6 +147,7 @@
     "it-tar": "1.2.2",
     "lru-cache": "5.1.1",
     "merge-options": "2.0.0",
+    "mortice": "2.0.0",
     "multiaddr": "7.4.3",
     "multiaddr-to-uri": "5.0.0",
     "p-memoize": "4.0.0",
diff --git a/scripts/prom-client-stub/index.js b/scripts/prom-client-stub/index.js
@@ -0,0 +1,12 @@
+'use strict'
+
+// This is a basic stub to build node-like js-ipfs
+// for use in Brave, but without prometheus libs which are not compatible
+// with browser context
+function Gauge () {}
+module.exports = {
+  Gauge,
+  register: {
+    clear () { }
+  }
+}
diff --git a/yarn.lock b/yarn.lock
@@ -10069,7 +10069,7 @@ moment@^2.10.6:
   resolved "https://registry.yarnpkg.com/moment/-/moment-2.27.0.tgz#8bff4e3e26a236220dfe3e36de756b6ebaa0105d"
   integrity sha512-al0MUK7cpIcglMv3YF13qSgdAIqxHTO7brRtaz3DlSULbqfazqkc5kEjNrLDOM7fsjshoFIihnU8snrP7zUvhQ==
 
-mortice@^2.0.0:
+mortice@2.0.0, mortice@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/mortice/-/mortice-2.0.0.tgz#7be171409c2115561ba3fc035e4527f9082eefde"
   integrity sha512-rXcjRgv2MRhpwGHErxKcDcp5IoA9CPvPFLXmmseQYIuQ2fSVu8tsMKi/eYUXzp/HH1s6y3IID/GwRqlSglDdRA==
@@ -11930,7 +11930,10 @@ progress@^2.0.0, progress@^2.0.1:
   resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8"
   integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==
 
-prom-client@^11.5.3, prom-client@^12.0.0, "prom-client@npm:empty-module@0.0.2", prometheus-gc-stats@^0.6.0, "prometheus-gc-stats@npm:empty-module@0.0.2", "v8@npm:empty-module@0.0.2":
+prom-client@^11.5.3, prom-client@^12.0.0, "prom-client@file:./scripts/prom-client-stub":
+  version "0.0.0"
+
+prometheus-gc-stats@^0.6.0, "prometheus-gc-stats@npm:empty-module@0.0.2", "v8@npm:empty-module@0.0.2":
   version "0.0.2"
   resolved "https://registry.yarnpkg.com/empty-module/-/empty-module-0.0.2.tgz#13b4dd8d4afeddd35e50c18dcd788c51087f1545"
   integrity sha1-E7TdjUr+3dNeUMGNzXiMUQh/FUU=

Original file line number	Diff line number	Diff line change
`@@ -11,7 +11,7 @@ function statusEntry ({ label, labelLegend, title, value, check, valueClass = ''`
`11`	`11`	`value = value \|\| value === 0 ? value : offline`
`12`	`12`	return html`
`13`	`13`	`<div title="${labelLegend}" class="ma0 pa0" style="line-height: 0.25">`
`14`		`- <span class="f7 tr monospace force-select-all ${valueClass}" title="${title}">${value}</span>`
	`14`	`+ <span class="f7 tr monospace force-select-all ${valueClass}" title="${title}">${value.substring(0, 13)}</span>`
`15`	`15`	`</div>`
`16`	`16`	`
`17`	`17`	`}`