From 2725fc5f424b7a5e2c3daa6aaf92dfc3f2352113 Mon Sep 17 00:00:00 2001 From: VASHISTH Chandramouli Date: Fri, 12 Sep 2025 17:07:59 +0530 Subject: [PATCH 1/4] implemented keyset pagination for gitlab #8529 Signed-off-by: VASHISTH Chandramouli --- .../plugins/gitlab/tasks/account_collector.go | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/backend/plugins/gitlab/tasks/account_collector.go b/backend/plugins/gitlab/tasks/account_collector.go index 747c5d6d286..6ca8f935acd 100644 --- a/backend/plugins/gitlab/tasks/account_collector.go +++ b/backend/plugins/gitlab/tasks/account_collector.go @@ -62,6 +62,8 @@ func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error { urlTemplate = "/users" } + var lastID int + collector, err := api.NewApiCollector(api.ApiCollectorArgs{ RawDataSubTaskArgs: *rawDataSubTaskArgs, ApiClient: data.ApiClient, @@ -69,6 +71,16 @@ func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error { PageSize: 100, Query: func(reqData *api.RequestData) (url.Values, errors.Error) { query := url.Values{} + if urlTemplate == "/users" { + query.Set("pagination", "keyset") + query.Set("order_by", "id") + query.Set("sort", "asc") + query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size)) + if lastID > 0 { + query.Set("id_after", fmt.Sprintf("%d", lastID)) + } + return query, nil + } query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page)) query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size)) return query, nil @@ -80,6 +92,15 @@ func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error { if err != nil { return nil, err } + if len(items) > 0 && urlTemplate == "/users" { + var tail struct { + ID int `json:"id"` + } + _ = json.Unmarshal(items[len(items)-1], &tail) + if tail.ID > 0 { + lastID = tail.ID + } + } return items, nil }, }) From 6db95157deddd46a8a6a3d809b52808fc2ca8d05 Mon Sep 17 00:00:00 2001 From: VASHISTH Chandramouli Date: Thu, 18 Sep 2025 10:41:39 +0530 Subject: [PATCH 2/4] keyset pagination on /users for >=16.5; fallback to offset --- backend/plugins/gitlab/tasks/account_collector.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/backend/plugins/gitlab/tasks/account_collector.go b/backend/plugins/gitlab/tasks/account_collector.go index 6ca8f935acd..2e2d5f004a3 100644 --- a/backend/plugins/gitlab/tasks/account_collector.go +++ b/backend/plugins/gitlab/tasks/account_collector.go @@ -36,6 +36,7 @@ func init() { } const RAW_USER_TABLE = "gitlab_api_users" +const KEYSET_MIN_VERSION = "v16.5.0" var CollectAccountsMeta = plugin.SubTaskMeta{ Name: "Collect Users", @@ -62,6 +63,14 @@ func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error { urlTemplate = "/users" } + useKeyset := false + if urlTemplate == "/users" && semver.IsValid(apiVersion) && semver.Compare(data.ApiClient.GetData(models.GitlabApiClientData_ApiVersion).(string), KEYSET_MIN_VERSION)>= 0 { + useKeyset = true + } else if urlTemplate == "/users" && !semver.IsValid(apiVersion) { + // If version unknown, be conservative for CE 11–16.4: default to offset + logger.Debug("GitLab version is unknown/invalid; falling back to offset pagination for /users") + } + var lastID int collector, err := api.NewApiCollector(api.ApiCollectorArgs{ @@ -71,7 +80,8 @@ func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error { PageSize: 100, Query: func(reqData *api.RequestData) (url.Values, errors.Error) { query := url.Values{} - if urlTemplate == "/users" { + // Use keyset only when gated true and only on /users endpoint + if useKeyset && urlTemplate == "/users" { query.Set("pagination", "keyset") query.Set("order_by", "id") query.Set("sort", "asc") From 376c231d56491a46527f694c0fb846d844964a18 Mon Sep 17 00:00:00 2001 From: VASHISTH Chandramouli Date: Mon, 22 Sep 2025 10:33:44 +0530 Subject: [PATCH 3/4] feat(plugin): fixed undefined:apiVersion issue --- backend/plugins/gitlab/tasks/account_collector.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/plugins/gitlab/tasks/account_collector.go b/backend/plugins/gitlab/tasks/account_collector.go index 2e2d5f004a3..6c5cbfeed90 100644 --- a/backend/plugins/gitlab/tasks/account_collector.go +++ b/backend/plugins/gitlab/tasks/account_collector.go @@ -63,8 +63,10 @@ func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error { urlTemplate = "/users" } + apiVersion := data.ApiClient.GetData(models.GitlabApiClientData_ApiVersion).(string) + useKeyset := false - if urlTemplate == "/users" && semver.IsValid(apiVersion) && semver.Compare(data.ApiClient.GetData(models.GitlabApiClientData_ApiVersion).(string), KEYSET_MIN_VERSION)>= 0 { + if urlTemplate == "/users" && semver.IsValid(apiVersion) && semver.Compare(apiVersion, KEYSET_MIN_VERSION)>= 0 { useKeyset = true } else if urlTemplate == "/users" && !semver.IsValid(apiVersion) { // If version unknown, be conservative for CE 11–16.4: default to offset From 1068c67d9c09ed69d34e20df79dc1ae108517bd7 Mon Sep 17 00:00:00 2001 From: VASHISTH Chandramouli Date: Wed, 1 Oct 2025 13:10:50 +0530 Subject: [PATCH 4/4] fix(plugin): fixed golangci-lint issue --- .../plugins/gitlab/tasks/account_collector.go | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/backend/plugins/gitlab/tasks/account_collector.go b/backend/plugins/gitlab/tasks/account_collector.go index 6c5cbfeed90..8b61a72032d 100644 --- a/backend/plugins/gitlab/tasks/account_collector.go +++ b/backend/plugins/gitlab/tasks/account_collector.go @@ -66,14 +66,14 @@ func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error { apiVersion := data.ApiClient.GetData(models.GitlabApiClientData_ApiVersion).(string) useKeyset := false - if urlTemplate == "/users" && semver.IsValid(apiVersion) && semver.Compare(apiVersion, KEYSET_MIN_VERSION)>= 0 { + if urlTemplate == "/users" && semver.IsValid(apiVersion) && semver.Compare(apiVersion, KEYSET_MIN_VERSION) >= 0 { useKeyset = true } else if urlTemplate == "/users" && !semver.IsValid(apiVersion) { // If version unknown, be conservative for CE 11–16.4: default to offset logger.Debug("GitLab version is unknown/invalid; falling back to offset pagination for /users") } - var lastID int + var lastID int collector, err := api.NewApiCollector(api.ApiCollectorArgs{ RawDataSubTaskArgs: *rawDataSubTaskArgs, @@ -84,15 +84,15 @@ func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error { query := url.Values{} // Use keyset only when gated true and only on /users endpoint if useKeyset && urlTemplate == "/users" { - query.Set("pagination", "keyset") - query.Set("order_by", "id") - query.Set("sort", "asc") - query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size)) - if lastID > 0 { - query.Set("id_after", fmt.Sprintf("%d", lastID)) - } - return query, nil - } + query.Set("pagination", "keyset") + query.Set("order_by", "id") + query.Set("sort", "asc") + query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size)) + if lastID > 0 { + query.Set("id_after", fmt.Sprintf("%d", lastID)) + } + return query, nil + } query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page)) query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size)) return query, nil @@ -105,14 +105,14 @@ func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error { return nil, err } if len(items) > 0 && urlTemplate == "/users" { - var tail struct { - ID int `json:"id"` - } - _ = json.Unmarshal(items[len(items)-1], &tail) - if tail.ID > 0 { - lastID = tail.ID - } - } + var tail struct { + ID int `json:"id"` + } + _ = json.Unmarshal(items[len(items)-1], &tail) + if tail.ID > 0 { + lastID = tail.ID + } + } return items, nil }, })