fix: use retry and throttle octokit plugins (#487)

Co-authored-by: Jonas Pauthier <[email protected]>
Co-authored-by: Gregor Martynus <[email protected]>

Author: 3 people

lib/add-channel.js

@@ -14,7 +14,7 @@ module.exports = async (pluginConfig, context) => {
   } = context;
   const {githubToken, githubUrl, githubApiPathPrefix, proxy} = resolveConfig(pluginConfig, context);
   const {owner, repo} = parseGithubUrl(repositoryUrl);
-  const github = getClient({githubToken, githubUrl, githubApiPathPrefix, proxy});
+  const octokit = getClient({githubToken, githubUrl, githubApiPathPrefix, proxy});
   let releaseId;
 
   const release = {owner, repo, name, prerelease: isPrerelease(branch), tag_name: gitTag};
@@ -24,14 +24,14 @@ module.exports = async (pluginConfig, context) => {
   try {
     ({
       data: {id: releaseId},
-    } = await github.repos.getReleaseByTag({owner, repo, tag: gitTag}));
+    } = await octokit.request('GET /repos/{owner}/{repo}/releases/tags/{tag}', {owner, repo, tag: gitTag}));
   } catch (error) {
     if (error.status === 404) {
       logger.log('There is no release for tag %s, creating a new one', gitTag);
 
       const {
         data: {html_url: url},
-      } = await github.repos.createRelease({...release, body: notes});
+      } = await octokit.request('POST /repos/{owner}/{repo}/releases', {...release, body: notes});
 
       logger.log('Published GitHub release: %s', url);
       return {url, name: RELEASE_NAME};
@@ -44,7 +44,7 @@ module.exports = async (pluginConfig, context) => {
 
   const {
     data: {html_url: url},
-  } = await github.repos.updateRelease({...release, release_id: releaseId});
+  } = await octokit.request('PATCH /repos/{owner}/{repo}/releases/{release_id}', {...release, release_id: releaseId});
 
   logger.log('Updated GitHub release: %s', url);
 

lib/definitions/rate-limit.js

@@ -0,0 +1,10 @@
+/**
+ * Default exponential backoff configuration for retries.
+ */
+const RETRY_CONF = {
+  // By default, Octokit does not retry on 404s.
+  // But we want to retry on 404s to account for replication lag.
+  doNotRetry: [400, 401, 403, 422],
+};
+
+module.exports = {RETRY_CONF};

lib/definitions/retry.js

@@ -0,0 +1,7 @@
+/**
+ * Default configuration for throttle.
+ * @see https://github.com/octokit/plugin-throttling.js#options
+ */
+const THROTTLE_CONF = {};
+
+module.exports = {THROTTLE_CONF};

lib/definitions/throttle.js

@@ -22,26 +22,34 @@ module.exports = async (pluginConfig, context) => {
   if (failComment === false || failTitle === false) {
     logger.log('Skip issue creation.');
   } else {
-    const github = getClient({githubToken, githubUrl, githubApiPathPrefix, proxy});
+    const octokit = getClient({githubToken, githubUrl, githubApiPathPrefix, proxy});
     // In case the repo changed name, get the new `repo`/`owner` as the search API will not follow redirects
-    const [owner, repo] = (await github.repos.get(parseGithubUrl(repositoryUrl))).data.full_name.split('/');
+    const {data: repoData} = await octokit.request('GET /repos/{owner}/{repo}', parseGithubUrl(repositoryUrl));
+    const [owner, repo] = repoData.full_name.split('/');
     const body = failComment ? template(failComment)({branch, errors}) : getFailComment(branch, errors);
-    const [srIssue] = await findSRIssues(github, failTitle, owner, repo);
+    const [srIssue] = await findSRIssues(octokit, failTitle, owner, repo);
 
     if (srIssue) {
       logger.log('Found existing semantic-release issue #%d.', srIssue.number);
       const comment = {owner, repo, issue_number: srIssue.number, body};
       debug('create comment: %O', comment);
       const {
         data: {html_url: url},
-      } = await github.issues.createComment(comment);
+      } = await octokit.request('POST /repos/{owner}/{repo}/issues/{issue_number}/comments', comment);
       logger.log('Added comment to issue #%d: %s.', srIssue.number, url);
     } else {
-      const newIssue = {owner, repo, title: failTitle, body: `${body}\n\n${ISSUE_ID}`, labels: labels || [], assignees};
+      const newIssue = {
+        owner,
+        repo,
+        title: failTitle,
+        body: `${body}\n\n${ISSUE_ID}`,
+        labels: labels || [],
+        assignees,
+      };
       debug('create issue: %O', newIssue);
       const {
         data: {html_url: url, number},
-      } = await github.issues.create(newIssue);
+      } = await octokit.request('POST /repos/{owner}/{repo}/issues', newIssue);
       logger.log('Created issue #%d: %s.', number, url);
     }
   }

lib/fail.js

@@ -1,9 +1,9 @@
 const {ISSUE_ID} = require('./definitions/constants');
 
-module.exports = async (github, title, owner, repo) => {
+module.exports = async (octokit, title, owner, repo) => {
   const {
     data: {items: issues},
-  } = await github.search.issuesAndPullRequests({
+  } = await octokit.request('GET /search/issues', {
     q: `in:title+repo:${owner}/${repo}+type:issue+state:open+${title}`,
   });
 

lib/find-sr-issues.js

@@ -1,35 +1,12 @@
-const {memoize, get} = require('lodash');
-const {Octokit} = require('@octokit/rest');
-const pRetry = require('p-retry');
-const Bottleneck = require('bottleneck');
 const urljoin = require('url-join');
 const HttpProxyAgent = require('http-proxy-agent');
 const HttpsProxyAgent = require('https-proxy-agent');
 
-const {RETRY_CONF, RATE_LIMITS, GLOBAL_RATE_LIMIT} = require('./definitions/rate-limit');
-
-/**
- * Http error status for which to not retry.
- */
-const SKIP_RETRY_CODES = new Set([400, 401, 403]);
-
-/**
- * Create or retrieve the throttler function for a given rate limit group.
- *
- * @param {Array} rate The rate limit group.
- * @param {String} limit The rate limits per API endpoints.
- * @param {Bottleneck} globalThrottler The global throttler.
- *
- * @return {Bottleneck} The throller function for the given rate limit group.
- */
-const getThrottler = memoize((rate, globalThrottler) =>
-  new Bottleneck({minTime: get(RATE_LIMITS, rate)}).chain(globalThrottler)
-);
+const SemanticReleaseOctokit = require('./semantic-release-octokit');
 
 module.exports = ({githubToken, githubUrl, githubApiPathPrefix, proxy}) => {
   const baseUrl = githubUrl && urljoin(githubUrl, githubApiPathPrefix);
-  const globalThrottler = new Bottleneck({minTime: GLOBAL_RATE_LIMIT});
-  const github = new Octokit({
+  const octokit = new SemanticReleaseOctokit({
     auth: `token ${githubToken}`,
     baseUrl,
     request: {
@@ -41,23 +18,5 @@ module.exports = ({githubToken, githubUrl, githubApiPathPrefix, proxy}) => {
     },
   });
 
-  github.hook.wrap('request', (request, options) => {
-    const access = options.method === 'GET' ? 'read' : 'write';
-    const rateCategory = options.url.startsWith('/search') ? 'search' : 'core';
-    const limitKey = [rateCategory, RATE_LIMITS[rateCategory][access] && access].filter(Boolean).join('.');
-
-    return pRetry(async () => {
-      try {
-        return await getThrottler(limitKey, globalThrottler).wrap(request)(options);
-      } catch (error) {
-        if (SKIP_RETRY_CODES.has(error.status)) {
-          throw new pRetry.AbortError(error);
-        }
-
-        throw error;
-      }
-    }, RETRY_CONF);
-  });
-
-  return github;
+  return octokit;
 };

lib/get-client.js

@@ -20,7 +20,7 @@ module.exports = async (pluginConfig, context) => {
   } = context;
   const {githubToken, githubUrl, githubApiPathPrefix, proxy, assets} = resolveConfig(pluginConfig, context);
   const {owner, repo} = parseGithubUrl(repositoryUrl);
-  const github = getClient({githubToken, githubUrl, githubApiPathPrefix, proxy});
+  const octokit = getClient({githubToken, githubUrl, githubApiPathPrefix, proxy});
   const release = {
     owner,
     repo,
@@ -37,7 +37,7 @@ module.exports = async (pluginConfig, context) => {
   if (!assets || assets.length === 0) {
     const {
       data: {html_url: url, id: releaseId},
-    } = await github.repos.createRelease(release);
+    } = await octokit.request('POST /repos/{owner}/{repo}/releases', release);
 
     logger.log('Published GitHub release: %s', url);
     return {url, name: RELEASE_NAME, id: releaseId};
@@ -49,7 +49,7 @@ module.exports = async (pluginConfig, context) => {
 
   const {
     data: {upload_url: uploadUrl, id: releaseId},
-  } = await github.repos.createRelease(draftRelease);
+  } = await octokit.request('POST /repos/{owner}/{repo}/releases', draftRelease);
 
   // Append assets to the release
   const globbedAssets = await globAssets(context, assets);
@@ -74,6 +74,7 @@ module.exports = async (pluginConfig, context) => {
 
       const fileName = template(asset.name || path.basename(filePath))(context);
       const upload = {
+        method: 'POST',
         url: uploadUrl,
         data: await readFile(path.resolve(cwd, filePath)),
         name: fileName,
@@ -92,14 +93,19 @@ module.exports = async (pluginConfig, context) => {
 
       const {
         data: {browser_download_url: downloadUrl},
-      } = await github.repos.uploadReleaseAsset(upload);
+      } = await octokit.request(upload);
       logger.log('Published file %s', downloadUrl);
     })
   );
 
   const {
     data: {html_url: url},
-  } = await github.repos.updateRelease({owner, repo, release_id: releaseId, draft: false});
+  } = await octokit.request('PATCH /repos/{owner}/{repo}/releases/{release_id}', {
+    owner,
+    repo,
+    release_id: releaseId,
+    draft: false,
+  });
 
   logger.log('Published GitHub release: %s', url);
   return {url, name: RELEASE_NAME, id: releaseId};

lib/publish.js

@@ -0,0 +1,35 @@
+/* istanbul ignore file */
+
+// If maintaining @octokit/core and the separate plugins gets to cumbersome
+// then the `octokit` package can be used which has all these plugins included.
+// However the `octokit` package has a lot of other things we don't care about.
+// We use only the bits we need to minimize the size of the package.
+const {Octokit} = require('@octokit/core');
+const {paginateRest} = require('@octokit/plugin-paginate-rest');
+const {retry} = require('@octokit/plugin-retry');
+const {throttling} = require('@octokit/plugin-throttling');
+
+const {RETRY_CONF} = require('./definitions/retry');
+const {THROTTLE_CONF} = require('./definitions/throttle');
+const {version} = require('../package.json');
+
+const onRetry = (retryAfter, options, octokit, retryCount) => {
+  octokit.log.warn(`Request quota exhausted for request ${options.method} ${options.url}`);
+
+  if (retryCount <= RETRY_CONF.retries) {
+    octokit.log.debug(`Will retry after ${retryAfter}.`);
+    return true;
+  }
+};
+
+const SemanticReleaseOctokit = Octokit.plugin(paginateRest, retry, throttling).defaults({
+  userAgent: `@semantic-release/github v${version}`,
+  retry: RETRY_CONF,
+  throttle: {
+    ...THROTTLE_CONF,
+    onRateLimit: onRetry,
+    onSecondaryRateLimit: onRetry,
+  },
+});
+
+module.exports = SemanticReleaseOctokit;

lib/semantic-release-octokit.js

@@ -32,9 +32,10 @@ module.exports = async (pluginConfig, context) => {
     addReleases,
   } = resolveConfig(pluginConfig, context);
 
-  const github = getClient({githubToken, githubUrl, githubApiPathPrefix, proxy});
+  const octokit = getClient({githubToken, githubUrl, githubApiPathPrefix, proxy});
   // In case the repo changed name, get the new `repo`/`owner` as the search API will not follow redirects
-  const [owner, repo] = (await github.repos.get(parseGithubUrl(repositoryUrl))).data.full_name.split('/');
+  const {data: repoData} = await octokit.request('GET /repos/{owner}/{repo}', parseGithubUrl(repositoryUrl));
+  const [owner, repo] = repoData.full_name.split('/');
 
   const errors = [];
 
@@ -46,15 +47,27 @@ module.exports = async (pluginConfig, context) => {
     const shas = commits.map(({hash}) => hash);
 
     const searchQueries = getSearchQueries(`repo:${owner}/${repo}+type:pr+is:merged`, shas).map(
-      async (q) => (await github.search.issuesAndPullRequests({q})).data.items
+      async (q) => (await octokit.request('GET /search/issues', {q})).data.items
     );
 
-    const prs = await pFilter(
-      uniqBy(flatten(await Promise.all(searchQueries)), 'number'),
-      async ({number}) =>
-        (await github.pulls.listCommits({owner, repo, pull_number: number})).data.find(({sha}) => shas.includes(sha)) ||
-        shas.includes((await github.pulls.get({owner, repo, pull_number: number})).data.merge_commit_sha)
-    );
+    const searchQueriesResults = await Promise.all(searchQueries);
+    const uniqueSearchQueriesResults = uniqBy(flatten(searchQueriesResults), 'number');
+    const prs = await pFilter(uniqueSearchQueriesResults, async ({number}) => {
+      const commits = await octokit.paginate('GET /repos/{owner}/{repo}/pulls/{pull_number}/commits', {
+        owner,
+        repo,
+        pull_number: number,
+      });
+      const matchingCommit = commits.find(({sha}) => shas.includes(sha));
+      if (matchingCommit) return matchingCommit;
+
+      const {data: pullRequest} = await octokit.request('GET /repos/{owner}/{repo}/pulls/{pull_number}', {
+        owner,
+        repo,
+        pull_number: number,
+      });
+      return shas.includes(pullRequest.merge_commit_sha);
+    });
 
     debug(
       'found pull requests: %O',
@@ -87,17 +100,15 @@ module.exports = async (pluginConfig, context) => {
           debug('create comment: %O', comment);
           const {
             data: {html_url: url},
-          } = await github.issues.createComment(comment);
+          } = await octokit.request('POST /repos/{owner}/{repo}/issues/{issue_number}/comments', comment);
           logger.log('Added comment to issue #%d: %s', issue.number, url);
 
           if (releasedLabels) {
             const labels = releasedLabels.map((label) => template(label)(context));
-            // Don’t use .issues.addLabels for GHE < 2.16 support
-            // https://github.com/semantic-release/github/issues/138
-            await github.request('POST /repos/:owner/:repo/issues/:number/labels', {
+            await octokit.request('POST /repos/{owner}/{repo}/issues/{issue_number}/labels', {
               owner,
               repo,
-              number: issue.number,
+              issue_number: issue.number,
               data: labels,
             });
             logger.log('Added labels %O to issue #%d', labels, issue.number);
@@ -120,7 +131,7 @@ module.exports = async (pluginConfig, context) => {
   if (failComment === false || failTitle === false) {
     logger.log('Skip closing issue.');
   } else {
-    const srIssues = await findSRIssues(github, failTitle, owner, repo);
+    const srIssues = await findSRIssues(octokit, failTitle, owner, repo);
 
     debug('found semantic-release issues: %O', srIssues);
 
@@ -132,7 +143,7 @@ module.exports = async (pluginConfig, context) => {
           debug('closing issue: %O', updateIssue);
           const {
             data: {html_url: url},
-          } = await github.issues.update(updateIssue);
+          } = await octokit.request('PATCH /repos/{owner}/{repo}/issues/{issue_number}', updateIssue);
           logger.log('Closed issue #%d: %s.', issue.number, url);
         } catch (error) {
           errors.push(error);
@@ -153,7 +164,12 @@ module.exports = async (pluginConfig, context) => {
           addReleases === 'top'
             ? additionalReleases.concat('\n---\n', nextRelease.notes)
             : nextRelease.notes.concat('\n---\n', additionalReleases);
-        await github.repos.updateRelease({owner, repo, release_id: ghRelaseId, body: newBody});
+        await octokit.request('PATCH /repos/{owner}/{repo}/releases/{release_id}', {
+          owner,
+          repo,
+          release_id: ghRelaseId,
+          body: newBody,
+        });
       }
     }
   }