refactor: replace custom fetch with ky-universal

This change simplifies code related to ad-hoc HTTP requests
by switching from custom code to ky-universal.

It is a drop-in replacement for fetch that works in browser and node
that retries failed request

License: MIT
Signed-off-by: Marcin Rataj <[email protected]>

Author: lidel

.aegir.js

@@ -8,7 +8,7 @@ const ipfsdServer = IPFSFactory.createServer()
 const preloadNode = MockPreloadNode.createNode()
 
 module.exports = {
-  bundlesize: { maxSize: '690kB' },
+  bundlesize: { maxSize: '692kB' },
   webpack: {
     resolve: {
       mainFields: ['browser', 'main'],

package.json

@@ -19,7 +19,6 @@
     "./src/core/runtime/add-from-fs-nodejs.js": "./src/core/runtime/add-from-fs-browser.js",
     "./src/core/runtime/config-nodejs.js": "./src/core/runtime/config-browser.js",
     "./src/core/runtime/dns-nodejs.js": "./src/core/runtime/dns-browser.js",
-    "./src/core/runtime/fetch-nodejs.js": "./src/core/runtime/fetch-browser.js",
     "./src/core/runtime/libp2p-nodejs.js": "./src/core/runtime/libp2p-browser.js",
     "./src/core/runtime/preload-nodejs.js": "./src/core/runtime/preload-browser.js",
     "./src/core/runtime/repo-nodejs.js": "./src/core/runtime/repo-browser.js",
@@ -121,6 +120,8 @@
     "just-flatten-it": "^2.1.0",
     "just-safe-set": "^2.1.0",
     "kind-of": "^6.0.2",
+    "ky": "~0.11.2",
+    "ky-universal": "~0.2.2",
     "libp2p": "~0.25.4",
     "libp2p-bootstrap": "~0.9.3",
     "libp2p-crypto": "~0.16.0",
@@ -146,7 +147,6 @@
     "multicodec": "~0.5.1",
     "multihashes": "~0.4.14",
     "multihashing-async": "~0.6.0",
-    "node-fetch": "^2.3.0",
     "p-queue": "^6.1.0",
     "peer-book": "~0.9.0",
     "peer-id": "~0.12.3",

src/core/components/files-regular/add-from-url.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const { URL } = require('iso-url')
-const fetch = require('../../runtime/fetch-nodejs')
+const { default: ky } = require('ky-universal')
 
 module.exports = (self) => {
   return async (url, options, callback) => {
@@ -14,7 +14,10 @@ module.exports = (self) => {
 
     try {
       const parsedUrl = new URL(url)
-      const res = await fetch(url)
+      const res = await ky(url, {
+        timeout: 15000,
+        retry: 3
+      })
 
       if (!res.ok) {
         throw new Error('unexpected status code: ' + res.status)

src/core/runtime/dns-browser.js

@@ -3,6 +3,7 @@
 
 const TLRU = require('../../utils/tlru')
 const { default: PQueue } = require('p-queue')
+const { default: ky } = require('ky-universal')
 
 // Avoid sending multiple queries for the same hostname by caching results
 const cache = new TLRU(1000)
@@ -46,12 +47,9 @@ module.exports = (domain, opts, callback) => {
     url += `&${encodeURIComponent(prop)}=${encodeURIComponent(opts[prop])}`
   })
 
-  _httpQueue.add(() => fetch(url, { mode: 'cors' })
-    .then((response) => response.json())
-    .then((response) => {
-      cache.set(query, response, ttl)
-      setImmediate(() => unpackResponse(domain, response, callback))
-    })
-    .catch((err) => setImmediate(() => callback(err)))
-  )
+  _httpQueue.add(async () => {
+    const response = await ky(url, { mode: 'cors' }).json()
+    cache.set(query, response, ttl)
+    setImmediate(() => unpackResponse(domain, response, callback))
+  }).catch((err) => setImmediate(() => callback(err)))
 }

src/core/runtime/fetch-browser.js

@@ -2,6 +2,7 @@
 'use strict'
 
 const { default: PQueue } = require('p-queue')
+const { default: ky } = require('ky-universal')
 const debug = require('debug')
 
 const log = debug('ipfs:preload')
@@ -17,15 +18,14 @@ module.exports = function preload (url, callback) {
   const controller = new AbortController()
   const signal = controller.signal
 
-  _httpQueue.add(() => fetch(url, { signal })
-    .then(res => {
-      if (!res.ok) {
-        log.error('failed to preload', url, res.status, res.statusText)
-        throw new Error(`failed to preload ${url}`)
-      }
-      setImmediate(callback)
-    })
-  ).catch((err) => setImmediate(() => callback(err)))
+  _httpQueue.add(async () => {
+    const res = await ky(url, { signal })
+    if (!res.ok) {
+      log.error('failed to preload', url, res.status, res.statusText)
+      throw new Error(`failed to preload ${url}`)
+    }
+    setImmediate(callback)
+  }).catch((err) => setImmediate(() => callback(err)))
 
   return {
     cancel: () => controller.abort()