feat: use keyset pagination for retrieving CI jobs

With xanzy/go-gitlab#1827, go-gitlab now supports keyset pagination, which is much more efficient for iterating through many pages of data. This commit switches the GitLab CI jobs API to use keyset pagination.
mvisonneau · Nov 13, 2023 · 050ea5c · 050ea5c
1 parent 66bcb68
commit 050ea5c
Show file tree

Hide file tree

Showing 2 changed files with 51 additions and 24 deletions.
diff --git a/pkg/gitlab/jobs.go b/pkg/gitlab/jobs.go
@@ -63,17 +63,21 @@ func (c *Client) ListPipelineJobs(ctx context.Context, projectNameOrID string, p
 		resp      *goGitlab.Response
 	)
 
-	options := &goGitlab.ListJobsOptions{
+	opt := &goGitlab.ListJobsOptions{
 		ListOptions: goGitlab.ListOptions{
-			Page:    1,
-			PerPage: 100,
+			OrderBy:    "id",
+			Pagination: "keyset",
+			PerPage:    100,
+			Sort:       "asc",
 		},
 	}
 
+	options := []goGitlab.RequestOptionFunc{goGitlab.WithContext(ctx)}
+
 	for {
 		c.rateLimit(ctx)
 
-		foundJobs, resp, err = c.Jobs.ListPipelineJobs(projectNameOrID, pipelineID, options, goGitlab.WithContext(ctx))
+		foundJobs, resp, err = c.Jobs.ListPipelineJobs(projectNameOrID, pipelineID, opt, options...)
 		if err != nil {
 			return
 		}
@@ -84,7 +88,7 @@ func (c *Client) ListPipelineJobs(ctx context.Context, projectNameOrID string, p
 			jobs = append(jobs, schemas.NewJob(*job))
 		}
 
-		if resp.CurrentPage >= resp.NextPage {
+		if resp.NextLink == "" {
 			log.WithFields(
 				log.Fields{
 					"project-name-or-id": projectNameOrID,
@@ -96,7 +100,10 @@ func (c *Client) ListPipelineJobs(ctx context.Context, projectNameOrID string, p
 			break
 		}
 
-		options.Page = resp.NextPage
+		options = []goGitlab.RequestOptionFunc{
+			goGitlab.WithContext(ctx),
+			goGitlab.WithKeysetPaginationParameters(resp.NextLink),
+		}
 	}
 
 	return
@@ -114,17 +121,21 @@ func (c *Client) ListPipelineBridges(ctx context.Context, projectNameOrID string
 		resp         *goGitlab.Response
 	)
 
-	options := &goGitlab.ListJobsOptions{
+	opt := &goGitlab.ListJobsOptions{
 		ListOptions: goGitlab.ListOptions{
-			Page:    1,
-			PerPage: 100,
+			OrderBy:    "id",
+			Pagination: "keyset",
+			PerPage:    100,
+			Sort:       "asc",
 		},
 	}
 
+	options := []goGitlab.RequestOptionFunc{goGitlab.WithContext(ctx)}
+
 	for {
 		c.rateLimit(ctx)
 
-		foundBridges, resp, err = c.Jobs.ListPipelineBridges(projectNameOrID, pipelineID, options, goGitlab.WithContext(ctx))
+		foundBridges, resp, err = c.Jobs.ListPipelineBridges(projectNameOrID, pipelineID, opt, options...)
 		if err != nil {
 			return
 		}
@@ -133,7 +144,7 @@ func (c *Client) ListPipelineBridges(ctx context.Context, projectNameOrID string
 
 		bridges = append(bridges, foundBridges...)
 
-		if resp.CurrentPage >= resp.NextPage {
+		if resp.NextLink == "" {
 			log.WithFields(
 				log.Fields{
 					"project-name-or-id": projectNameOrID,
@@ -145,7 +156,10 @@ func (c *Client) ListPipelineBridges(ctx context.Context, projectNameOrID string
 			break
 		}
 
-		options.Page = resp.NextPage
+		options = []goGitlab.RequestOptionFunc{
+			goGitlab.WithContext(ctx),
+			goGitlab.WithKeysetPaginationParameters(resp.NextLink),
+		}
 	}
 
 	return
@@ -233,17 +247,21 @@ func (c *Client) ListRefMostRecentJobs(ctx context.Context, ref schemas.Ref) (jo
 		resp      *goGitlab.Response
 	)
 
-	options := &goGitlab.ListJobsOptions{
+	opt := &goGitlab.ListJobsOptions{
 		ListOptions: goGitlab.ListOptions{
-			Page:    1,
-			PerPage: 100,
+			OrderBy:    "id",
+			Pagination: "keyset",
+			PerPage:    100,
+			Sort:       "asc",
 		},
 	}
 
+	options := []goGitlab.RequestOptionFunc{goGitlab.WithContext(ctx)}
+
 	for {
 		c.rateLimit(ctx)
 
-		foundJobs, resp, err = c.Jobs.ListProjectJobs(ref.Project.Name, options, goGitlab.WithContext(ctx))
+		foundJobs, resp, err = c.Jobs.ListProjectJobs(ref.Project.Name, opt, options...)
 		if err != nil {
 			return
 		}
@@ -272,7 +290,7 @@ func (c *Client) ListRefMostRecentJobs(ctx context.Context, ref schemas.Ref) (jo
 			}
 		}
 
-		if resp.CurrentPage >= resp.NextPage {
+		if resp.NextLink == "" {
 			var notFoundJobs []string
 
 			for k := range jobsToRefresh {
@@ -293,7 +311,10 @@ func (c *Client) ListRefMostRecentJobs(ctx context.Context, ref schemas.Ref) (jo
 			break
 		}
 
-		options.Page = resp.NextPage
+		options = []goGitlab.RequestOptionFunc{
+			goGitlab.WithContext(ctx),
+			goGitlab.WithKeysetPaginationParameters(resp.NextLink),
+		}
 	}
 
 	return

diff --git a/pkg/gitlab/jobs_test.go b/pkg/gitlab/jobs_test.go
@@ -81,8 +81,10 @@ func TestListPipelineJobs(t *testing.T) {
 		func(w http.ResponseWriter, r *http.Request) {
 			assert.Equal(t, "GET", r.Method)
 			expectedQueryParams := url.Values{
-				"page":     []string{"1"},
-				"per_page": []string{"100"},
+				"order_by":   []string{"id"},
+				"pagination": []string{"keyset"},
+				"per_page":   []string{"100"},
+				"sort":       []string{"asc"},
 			}
 			assert.Equal(t, expectedQueryParams, r.URL.Query())
 			fmt.Fprint(w, `[{"id":1},{"id":2}]`)
@@ -110,8 +112,10 @@ func TestListPipelineBridges(t *testing.T) {
 		func(w http.ResponseWriter, r *http.Request) {
 			assert.Equal(t, "GET", r.Method)
 			expectedQueryParams := url.Values{
-				"page":     []string{"1"},
-				"per_page": []string{"100"},
+				"order_by":   []string{"id"},
+				"pagination": []string{"keyset"},
+				"per_page":   []string{"100"},
+				"sort":       []string{"asc"},
 			}
 			assert.Equal(t, expectedQueryParams, r.URL.Query())
 			fmt.Fprint(w, `[{"id":1,"pipeline":{"id":100}}]`)
@@ -148,8 +152,10 @@ func TestListRefMostRecentJobs(t *testing.T) {
 		func(w http.ResponseWriter, r *http.Request) {
 			assert.Equal(t, "GET", r.Method)
 			expectedQueryParams := url.Values{
-				"page":     []string{"1"},
-				"per_page": []string{"100"},
+				"order_by":   []string{"id"},
+				"pagination": []string{"keyset"},
+				"per_page":   []string{"100"},
+				"sort":       []string{"asc"},
 			}
 			assert.Equal(t, expectedQueryParams, r.URL.Query())
 			fmt.Fprint(w, `[{"id":3,"name":"foo","ref":"yay"},{"id":4,"name":"bar","ref":"yay"}]`)