Deprecate request/requestretry – Switch to Axios/axios-retry, Add Central getStream Helper

lib/fetch.js

@@ -2,8 +2,10 @@
 // SPDX-License-Identifier: MIT
 
 const axios = require('axios')
+const axiosRetry = require('axios-retry')
 
 const defaultHeaders = Object.freeze({ 'User-Agent': 'clearlydefined.io crawler ([email protected])' })
+const maximumAttempts = 3
 
 axios.defaults.headers = defaultHeaders
 
@@ -14,12 +16,14 @@ function buildRequestOptions(request) {
   } else if (request.encoding === null) {
     responseType = 'stream'
   }
-
   const validateOptions = {}
   if (request.simple === false) {
     validateOptions.validateStatus = () => true
   }
-
+  if (request.gzip) {
+    if (!request.headers) request.headers = {}
+    request.headers['Accept-Encoding'] = 'gzip'
+  }
   return {
     method: request.method,
     url: request.url || request.uri,
@@ -49,4 +53,51 @@ function withDefaults(opts) {
   return request => callFetch(request, axiosInstance)
 }
 
-module.exports = { callFetch, withDefaults, defaultHeaders }
+async function getStream(opt) {
+  if (typeof opt === 'string') {
+    opt = { url: opt }
+  }
+  const request = {
+    ...opt,
+    encoding: null,
+    method: 'GET',
+    headers: { ...defaultHeaders, ...(opt.headers || {}) }
+  }
+  return await callFetch(request)
+}
+
+async function callFetchWithRetry(url, options = {}, retryOptions = {}) {
+  const { maxAttempts = maximumAttempts, ...otherRetryOpts } = retryOptions
+  const axiosInstance = axios.create()
+  axiosRetry(axiosInstance, { retries: maxAttempts, retryDelay: axiosRetry.exponentialDelay, ...otherRetryOpts })
+  options.resolveWithFullResponse ??= true
+  const request = { url, ...options }
+  try {
+    const response = await callFetch(request, axiosInstance)
+    if (!options.resolveWithFullResponse) return response
+    return {
+      statusCode: response.status,
+      headers: response.headers,
+      body: response.data,
+      request: response.request
+    }
+  } catch (err) {
+    if (err.response) {
+      return {
+        statusCode: err.response.status,
+        headers: err.response.headers,
+        body: err.response.data,
+        request: err.response.request
+      }
+    }
+    throw err
+  }
+}
+
+module.exports = {
+  callFetch,
+  callFetchWithRetry,
+  withDefaults,
+  defaultHeaders,
+  getStream
+}

lib/sourceDiscovery.js

@@ -3,7 +3,6 @@
 
 const { get, uniq, uniqWith } = require('lodash')
 const ghrequestor = require('ghrequestor')
-const request = require('requestretry')
 const geit = require('geit')
 // TODO why not parse-github-repo-url (10x more downloads)
 const parseGitHubUrl = require('parse-github-url')
@@ -75,7 +74,8 @@ async function discoverFromGitHubRefs(version, candidate, options) {
       headers,
       maxAttempts: 3,
       retryDelay: 250,
-      retryStrategy: request.RetryStrategies.HTTPOrNetworkError,
+      // This is the default strategy in axios retry. We ghrequestor currently depends on requestretry, but we will replace it with axios retry soon.
+      // retryStrategy: request.RetryStrategies.HTTPOrNetworkError,
       tokenLowerBound: 10,
       json: true
     })

package.json

@@ -72,7 +72,6 @@
     "parse-github-url": "^1.0.2",
     "patch-package": "^6.5.1",
     "qlimit": "^0.1.1",
-    "requestretry": "^4.0.0",
     "semver": "^7.6.0",
     "sha.js": "^2.4.11",
     "spdx-correct": "^3.2.0",

providers/fetch/condaFetch.js

@@ -5,7 +5,7 @@ const AbstractFetch = require('./abstractFetch')
 const { clone } = require('lodash')
 const fs = require('fs')
 const memCache = require('memory-cache')
-const nodeRequest = require('request')
+const { getStream: nodeRequest } = require('../../lib/fetch')
 const FetchResult = require('../../lib/fetchResult')
 
 class CondaFetch extends AbstractFetch {
@@ -167,11 +167,16 @@ class CondaFetch extends AbstractFetch {
     return new Promise((resolve, reject) => {
       const options = { url: downloadUrl, headers: this.headers }
       nodeRequest
-        .get(options, (error, response) => {
-          if (error) return reject(error)
-          if (response.statusCode !== 200) return reject(new Error(`${response.statusCode} ${response.statusMessage}`))
+        .getStream(options)
+        .then(response => {
+          if (response.statusCode !== 200) {
+            return reject(new Error(`${response.statusCode} ${response.message}`))
+          }
+          response.pipe(fs.createWriteStream(destination)).on('finish', resolve)
+        })
+        .catch(error => {
+          return reject(error)
         })
-        .pipe(fs.createWriteStream(destination).on('finish', () => resolve()))
     })
   }
 

providers/fetch/debianFetch.js

@@ -8,7 +8,7 @@ const domain = require('domain')
 const fs = require('fs')
 const linebyline = require('linebyline')
 const memCache = require('memory-cache')
-const nodeRequest = require('request')
+const { getStream: nodeRequest } = require('../../lib/fetch')
 const path = require('path')
 const { promisify } = require('util')
 const { callFetch: requestPromise } = require('../../lib/fetch')
@@ -104,16 +104,17 @@ class DebianFetch extends AbstractFetch {
           return reject(error)
         })
         dom.run(() => {
-          nodeRequest
-            .get(packageFileMap.url)
-            .pipe(bz2())
-            .pipe(fs.createWriteStream(this.packageMapFileLocation))
-            .on('finish', () => {
-              this.logger.info(
-                `Debian: retrieved ${packageFileMap.url}. Stored map file at ${this.packageMapFileLocation}`
-              )
-              return resolve()
-            })
+          nodeRequest(packageFileMap.url).then(response => {
+            response
+              .pipe(bz2())
+              .pipe(fs.createWriteStream(this.packageMapFileLocation))
+              .on('finish', () => {
+                this.logger.info(
+                  `Debian: retrieved ${packageFileMap.url}. Stored map file at ${this.packageMapFileLocation}`
+                )
+                return resolve()
+              })
+          })
         })
       })
     }
@@ -214,14 +215,16 @@ class DebianFetch extends AbstractFetch {
       const dom = domain.create()
       dom.on('error', error => reject(error))
       dom.run(() => {
-        nodeRequest
-          .get(downloadUrl, (error, response) => {
-            if (error) return reject(error)
-            if (response.statusCode !== 200)
-              return reject(new Error(`${response.statusCode} ${response.statusMessage}`))
+        nodeRequest(downloadUrl)
+          .then(response => {
+            if (response.statusCode !== 200) {
+              return reject(new Error(`${response.statusCode} ${response.message}`))
+            }
+            response.pipe(fs.createWriteStream(destination)).on('finish', () => resolve())
+          })
+          .catch(error => {
+            return reject(error)
           })
-          .pipe(fs.createWriteStream(destination))
-          .on('finish', () => resolve())
       })
     })
   }

providers/fetch/goFetch.js

@@ -1,7 +1,7 @@
 const { clone } = require('lodash')
 const { callFetch: requestPromise } = require('../../lib/fetch')
 const AbstractFetch = require('./abstractFetch')
-const nodeRequest = require('request')
+const { getStream: nodeRequest } = require('../../lib/fetch')
 const fs = require('fs')
 const axios = require('axios')
 const { default: axiosRetry, exponentialDelay, isNetworkOrIdempotentRequestError } = require('axios-retry')
@@ -112,14 +112,16 @@ class GoFetch extends AbstractFetch {
     const url = this._buildUrl(spec)
 
     const status = await new Promise(resolve => {
-      nodeRequest
-        .get(url, (error, response) => {
-          if (error) this.logger.error(this._google_proxy_error_string(error))
+      nodeRequest(url)
+        .then(response => {
           if (response.statusCode !== 200) return resolve(false)
+          response.pipe(fs.createWriteStream(destination)).on('finish', () => resolve(true))
+        })
+        .catch(error => {
+          this.logger.error(this._google_proxy_error_string(error))
+          resolve(false)
         })
-        .pipe(fs.createWriteStream(destination).on('finish', () => resolve(true)))
     })
-
     if (status) return true
   }
 

providers/fetch/mavenBasedFetch.js

@@ -2,8 +2,7 @@
 // SPDX-License-Identifier: MIT
 
 const AbstractFetch = require('./abstractFetch')
-const { callFetch, defaultHeaders } = require('../../lib/fetch')
-const nodeRequest = require('request')
+const { callFetch, getStream } = require('../../lib/fetch')
 const { clone, get } = require('lodash')
 const { promisify } = require('util')
 const fs = require('fs')
@@ -28,7 +27,7 @@ class MavenBasedFetch extends AbstractFetch {
     super(options)
     this._providerMap = { ...providerMap }
     this._handleRequestPromise = options.requestPromise || callFetch
-    this._handleRequestStream = options.requestStream || nodeRequest.defaults({ headers: defaultHeaders }).get
+    this._handleRequestStream = options.requestStream || getStream
   }
 
   canHandle(request) {
@@ -93,10 +92,15 @@ class MavenBasedFetch extends AbstractFetch {
     for (let extension of extensions) {
       const url = this._buildUrl(spec, extension)
       const status = await new Promise(resolve => {
-        this._handleRequestStream(url, (error, response) => {
-          if (error) this.logger.error(error)
-          if (response.statusCode !== 200) return resolve(false)
-        }).pipe(fs.createWriteStream(destination).on('finish', () => resolve(true)))
+        this._handleRequestStream(url)
+          .then(response => {
+            if (response.statusCode !== 200) return resolve(false)
+            response.pipe(fs.createWriteStream(destination)).on('finish', () => resolve(true))
+          })
+          .catch(error => {
+            this.logger.error(error)
+            resolve(false)
+          })
       })
       if (status) return true
     }

providers/fetch/npmjsFetch.js

@@ -2,8 +2,7 @@
 // SPDX-License-Identifier: MIT
 
 const AbstractFetch = require('./abstractFetch')
-const nodeRequest = require('request')
-const { callFetch: requestPromise } = require('../../lib/fetch')
+const { callFetch: requestPromise, getStream: nodeRequest } = require('../../lib/fetch')
 const fs = require('fs')
 const { clone, get } = require('lodash')
 const FetchResult = require('../../lib/fetchResult')
@@ -42,12 +41,14 @@ class NpmFetch extends AbstractFetch {
 
   async _getPackage(spec, destination) {
     return new Promise((resolve, reject) => {
-      nodeRequest
-        .get(this._buildUrl(spec), (error, response) => {
-          if (error) return reject(error)
-          if (response.statusCode !== 200) reject(new Error(`${response.statusCode} ${response.statusMessage}`))
+      nodeRequest(this._buildUrl(spec))
+        .then(response => {
+          if (response.statusCode !== 200) reject(new Error(`${response.statusCode} ${response.message}`))
+          response.pipe(fs.createWriteStream(destination)).on('finish', () => resolve(true))
+        })
+        .catch(error => {
+          return reject(error)
         })
-        .pipe(fs.createWriteStream(destination).on('finish', () => resolve(null)))
     })
   }
 

providers/fetch/nugetFetch.js

@@ -7,7 +7,7 @@ const fs = require('fs')
 const mkdirp = require('mkdirp')
 const path = require('path')
 const { promisify } = require('util')
-const requestRetry = require('requestretry').defaults({ maxAttempts: 3, fullResponse: true })
+const { callFetchWithRetry: requestRetry } = require('../../lib/fetch')
 const FetchResult = require('../../lib/fetchResult')
 
 const providerMap = {
@@ -69,7 +69,7 @@ class NuGetFetch extends AbstractFetch {
     // https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource
     // Example: https://api.nuget.org/v3/registration5/moq/4.8.2.json and follow catalogEntry
     // https://api.nuget.org/v3/registration5-gz-semver2/microsoft.powershell.native/7.0.0-preview.1.json
-    const { body, statusCode } = await requestRetry.get(
+    const { body, statusCode } = await requestRetry(
       `${baseUrl}/v3/registration5-gz-semver2/${spec.name.toLowerCase()}/${spec.revision}.json`,
       { gzip: true }
     )
@@ -87,7 +87,7 @@ class NuGetFetch extends AbstractFetch {
     // https://docs.microsoft.com/en-us/nuget/api/package-base-address-resource
     // Example: https://api.nuget.org/v3-flatcontainer/moq/index.json
     const baseUrl = providerMap.nuget
-    const { body, statusCode } = await requestRetry.get(`${baseUrl}/v3-flatcontainer/${name}/index.json`, {
+    const { body, statusCode } = await requestRetry(`${baseUrl}/v3-flatcontainer/${name}/index.json`, {
       json: true
     })
     // If statusCode is not 200, XML may be returned
@@ -100,17 +100,18 @@ class NuGetFetch extends AbstractFetch {
 
   async _getPackage(zip, packageContentUrl) {
     return new Promise((resolve, reject) => {
-      requestRetry
-        .get(packageContentUrl, { json: false, encoding: null })
-        .pipe(fs.createWriteStream(zip))
-        .on('finish', () => resolve(null))
-        .on('error', reject)
+      requestRetry(packageContentUrl, { json: false, encoding: null }).then(response => {
+        response.body
+          .pipe(fs.createWriteStream(zip))
+          .on('finish', () => resolve(null))
+          .on('error', reject)
+      })
     })
   }
 
   async _getManifest(catalogEntryUrl) {
     // Example: https://api.nuget.org/v3/catalog0/data/2018.10.29.04.23.22/xunit.core.2.4.1.json
-    const { body, statusCode } = await requestRetry.get(catalogEntryUrl)
+    const { body, statusCode } = await requestRetry(catalogEntryUrl)
     if (statusCode !== 200) return null
     return JSON.parse(body)
   }
@@ -119,7 +120,7 @@ class NuGetFetch extends AbstractFetch {
   async _getNuspec(spec) {
     // https://docs.microsoft.com/en-us/nuget/api/package-base-address-resource#download-package-manifest-nuspec
     // Example: https://api.nuget.org/v3-flatcontainer/newtonsoft.json/11.0.1/newtonsoft.json.nuspec
-    const { body, statusCode } = await requestRetry.get(
+    const { body, statusCode } = await requestRetry(
       `https://api.nuget.org/v3-flatcontainer/${spec.name.toLowerCase()}/${spec.revision}/${spec.name.toLowerCase()}.nuspec`
     )
     if (statusCode !== 200) return null
@@ -146,7 +147,7 @@ class NuGetFetch extends AbstractFetch {
     if (licenseUrl.toLowerCase().includes('license_url_here_or_delete_this_line')) return
     const downloadedLicenseDirName = path.join(dirName, 'clearlydefined', 'downloaded')
     await promisify(mkdirp)(downloadedLicenseDirName)
-    const { body, statusCode } = await requestRetry.get(licenseUrl)
+    const { body, statusCode } = await requestRetry(licenseUrl)
     if (statusCode !== 200) return
     await promisify(fs.writeFile)(path.join(downloadedLicenseDirName, 'LICENSE'), body)
   }