Use alternative way of computing last page of search results

Author: robertbrignull

extensions/ql-vscode/src/common/github-api-response-helper.ts

@@ -0,0 +1,18 @@
+export function getLinkHeaderLastPage(
+  linkHeader: string | undefined,
+): number | undefined {
+  if (!linkHeader) {
+    return undefined;
+  }
+
+  const lastPage = linkHeader
+    .split(", ")
+    .find((link) => link.includes('rel="last"'))
+    ?.match(/&page=(\d+)/);
+
+  if (!lastPage) {
+    return undefined;
+  }
+
+  return parseInt(lastPage[1], 10);
+}

extensions/ql-vscode/src/databases/code-search-api.ts

@@ -8,6 +8,7 @@ import type { ProgressCallback } from "../common/vscode/progress";
 import { UserCancellationException } from "../common/vscode/progress";
 import type { EndpointDefaults } from "@octokit/types";
 import { getOctokitBaseUrl } from "../common/vscode/octokit";
+import { getLinkHeaderLastPage } from "../common/github-api-response-helper";
 
 export async function getCodeSearchRepositories(
   query: string,
@@ -29,9 +30,14 @@ export async function getCodeSearchRepositories(
   )) {
     i++;
     nwos.push(...response.data.map((item) => item.repository.full_name));
-    const totalNumberOfResultPages = Math.ceil(response.data.total_count / 100);
+
+    // If we can't find the last page, then default to 10 because that's
+    // the maximum number of pages we can get when using `per_page: 100`.
+    // This means the progress bar might complete earlier than expected
+    // but that's better experience for the user than underestimating.
     const totalNumberOfRequests =
-      totalNumberOfResultPages > 10 ? 10 : totalNumberOfResultPages;
+      getLinkHeaderLastPage(response.headers.link) ?? 10;
+
     progress({
       maxStep: totalNumberOfRequests,
       step: i,

extensions/ql-vscode/test/unit-tests/github-api-response-helper.test.ts

@@ -0,0 +1,27 @@
+import { getLinkHeaderLastPage } from "../../src/common/github-api-response-helper";
+
+describe("getLinkHeaderLastPage", () => {
+  it("should return undefined when passed undefined", () => {
+    expect(getLinkHeaderLastPage(undefined)).toBeUndefined();
+  });
+
+  it("should return undefined when passed an invalid string", () => {
+    expect(getLinkHeaderLastPage("foobar")).toBeUndefined();
+  });
+
+  it("should return undefined when passed a valid link header without a last page", () => {
+    expect(
+      getLinkHeaderLastPage(
+        '<https://api.github.com/search/code?q=foo+language%3Ajava&per_page=100&page=2>; rel="next"',
+      ),
+    ).toBeUndefined();
+  });
+
+  it("should parse last page number when passed a valid link header with a last page", () => {
+    expect(
+      getLinkHeaderLastPage(
+        '<https://api.github.com/search/code?q=foo+language%3Ajava&per_page=100&page=2>; rel="prev", <https://api.github.com/search/code?q=foo+language%3Ajava&per_page=100&page=4>; rel="next", <https://api.github.com/search/code?q=foo+language%3Ajava&per_page=100&page=8>; rel="last"',
+      ),
+    ).toBe(8);
+  });
+});