From 5ba00cfced6f2af25235b5958a35e175fad15563 Mon Sep 17 00:00:00 2001 From: Klesh Wong Date: Mon, 2 Sep 2024 21:18:29 +0800 Subject: [PATCH 01/23] feat: gitlab mr_extractor support incremental sync --- .../helpers/pluginhelper/api/api_rawdata.go | 2 +- .../pluginhelper/api/subtask_state_manager.go | 2 +- backend/impls/dalgorm/encdec_serializer.go | 5 +++++ backend/plugins/gitlab/tasks/mr_extractor.go | 22 +++++++++---------- backend/plugins/gitlab/tasks/shared.go | 19 +++++++++++++--- 5 files changed, 34 insertions(+), 16 deletions(-) diff --git a/backend/helpers/pluginhelper/api/api_rawdata.go b/backend/helpers/pluginhelper/api/api_rawdata.go index ef16dde6db8..5e9bb783529 100644 --- a/backend/helpers/pluginhelper/api/api_rawdata.go +++ b/backend/helpers/pluginhelper/api/api_rawdata.go @@ -34,7 +34,7 @@ type RawData struct { Data []byte Url string Input json.RawMessage `gorm:"type:json"` - CreatedAt time.Time + CreatedAt time.Time `gorm:"index"` } type TaskOptions interface { diff --git a/backend/helpers/pluginhelper/api/subtask_state_manager.go b/backend/helpers/pluginhelper/api/subtask_state_manager.go index 407a291bffb..7dd7d2370b0 100644 --- a/backend/helpers/pluginhelper/api/subtask_state_manager.go +++ b/backend/helpers/pluginhelper/api/subtask_state_manager.go @@ -34,7 +34,7 @@ type SubtaskCommonArgs struct { plugin.SubTaskContext Table string // raw table name Params any // for filtering rows belonging to the scope (jira board, github repo) of the subtask - SubtaskConfig any // for determining whether the subtask should run in incremental or full sync mode + SubtaskConfig any // for determining whether the subtask should run in Incremental or Full-Sync mode by comparing with the previous config to see if it changed BatchSize int // batch size for saving data } diff --git a/backend/impls/dalgorm/encdec_serializer.go b/backend/impls/dalgorm/encdec_serializer.go index bb658919590..fd7a9d88059 100644 --- a/backend/impls/dalgorm/encdec_serializer.go +++ b/backend/impls/dalgorm/encdec_serializer.go @@ -91,6 +91,11 @@ func (es *EncDecSerializer) Value(ctx context.Context, field *schema.Field, dst } target = string(b) } + if field.GORMDataType == "string" { + println("field.GORMDataType == string", field.Size) + gormTag, ok := field.Tag.Lookup("gorm") + println(ok, gormTag) + } return plugin.Encrypt(es.encryptionSecret, target) } diff --git a/backend/plugins/gitlab/tasks/mr_extractor.go b/backend/plugins/gitlab/tasks/mr_extractor.go index 76253833bc0..f2508052719 100644 --- a/backend/plugins/gitlab/tasks/mr_extractor.go +++ b/backend/plugins/gitlab/tasks/mr_extractor.go @@ -96,8 +96,8 @@ var ExtractApiMergeRequestsMeta = plugin.SubTaskMeta{ Dependencies: []*plugin.SubTaskMeta{&CollectApiMergeRequestsMeta}, } -func ExtractApiMergeRequests(taskCtx plugin.SubTaskContext) errors.Error { - rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_MERGE_REQUEST_TABLE) +func ExtractApiMergeRequests(subtaskCtx plugin.SubTaskContext) errors.Error { + subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_MERGE_REQUEST_TABLE) config := data.Options.ScopeConfig var labelTypeRegex *regexp.Regexp var labelComponentRegex *regexp.Regexp @@ -118,10 +118,14 @@ func ExtractApiMergeRequests(taskCtx plugin.SubTaskContext) errors.Error { } } - extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ - RawDataSubTaskArgs: *rawDataSubTaskArgs, - Extract: func(row *api.RawData) ([]interface{}, errors.Error) { + subtaskCommonArgs.SubtaskConfig = map[string]any{ + "prType": prType, + "prComponent": prComponent, + } + extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs{ + SubtaskCommonArgs: subtaskCommonArgs, + Extract: func(row *api.RawData) ([]interface{}, errors.Error) { mr := &MergeRequestRes{} s := string(row.Data) err := errors.Convert(json.Unmarshal(row.Data, mr)) @@ -142,7 +146,7 @@ func ExtractApiMergeRequests(taskCtx plugin.SubTaskContext) errors.Error { } } - results := make([]interface{}, 0, len(mr.Reviewers)+1) + results := make([]interface{}, 0, len(mr.Reviewers)+len(mr.Labels)+1) gitlabMergeRequest.ConnectionId = data.Options.ConnectionId results = append(results, gitlabMergeRequest) for _, label := range mr.Labels { @@ -188,17 +192,13 @@ func ExtractApiMergeRequests(taskCtx plugin.SubTaskContext) errors.Error { } results = append(results, gitlabAssignee) } - return results, nil }, }) - if err != nil { - return errors.Convert(err) + return err } - return extractor.Execute() - } func convertMergeRequest(mr *MergeRequestRes) (*models.GitlabMergeRequest, errors.Error) { diff --git a/backend/plugins/gitlab/tasks/shared.go b/backend/plugins/gitlab/tasks/shared.go index 4051c93b57b..976052968ff 100644 --- a/backend/plugins/gitlab/tasks/shared.go +++ b/backend/plugins/gitlab/tasks/shared.go @@ -155,10 +155,10 @@ func GetQuery(reqData *helper.RequestData) (url.Values, errors.Error) { return query, nil } -func CreateRawDataSubTaskArgs(taskCtx plugin.SubTaskContext, Table string) (*helper.RawDataSubTaskArgs, *GitlabTaskData) { - data := taskCtx.GetData().(*GitlabTaskData) +func CreateRawDataSubTaskArgs(subtaskCtx plugin.SubTaskContext, Table string) (*helper.RawDataSubTaskArgs, *GitlabTaskData) { + data := subtaskCtx.GetData().(*GitlabTaskData) rawDataSubTaskArgs := &helper.RawDataSubTaskArgs{ - Ctx: taskCtx, + Ctx: subtaskCtx, Params: models.GitlabApiParams{ ProjectId: data.Options.ProjectId, ConnectionId: data.Options.ConnectionId, @@ -168,6 +168,19 @@ func CreateRawDataSubTaskArgs(taskCtx plugin.SubTaskContext, Table string) (*hel return rawDataSubTaskArgs, data } +func CreateSubtaskCommonArgs(subtaskCtx plugin.SubTaskContext, table string) (*helper.SubtaskCommonArgs, *GitlabTaskData) { + data := subtaskCtx.GetData().(*GitlabTaskData) + args := &helper.SubtaskCommonArgs{ + SubTaskContext: subtaskCtx, + Table: table, + Params: models.GitlabApiParams{ + ConnectionId: data.Options.ConnectionId, + ProjectId: data.Options.ProjectId, + }, + } + return args, data +} + func GetMergeRequestsIterator(taskCtx plugin.SubTaskContext, apiCollector *helper.StatefulApiCollector) (*helper.DalCursorIterator, errors.Error) { db := taskCtx.GetDal() data := taskCtx.GetData().(*GitlabTaskData) From 9932391b14045a403022e7c7f1137f9e526bc663 Mon Sep 17 00:00:00 2001 From: Klesh Wong Date: Tue, 3 Sep 2024 19:58:41 +0800 Subject: [PATCH 02/23] feat: incr mode transformation support for deployment/issue/job/mr and others --- .../pluginhelper/api/enrich_with_regex.go | 8 +++ .../gitlab/tasks/deployment_convertor.go | 40 ++++++------ .../gitlab/tasks/deployment_extractor.go | 6 +- .../gitlab/tasks/issue_assignee_convertor.go | 47 +++++++------- .../plugins/gitlab/tasks/issue_convertor.go | 44 +++++++------ .../plugins/gitlab/tasks/issue_extractor.go | 12 +++- .../gitlab/tasks/issue_label_convertor.go | 49 ++++++++------- backend/plugins/gitlab/tasks/job_convertor.go | 40 ++++++------ backend/plugins/gitlab/tasks/job_extractor.go | 6 +- .../gitlab/tasks/mr_assignee_convertor.go | 50 +++++++-------- .../gitlab/tasks/mr_commit_convertor.go | 62 ++++++++----------- .../gitlab/tasks/mr_commit_extractor.go | 8 +-- backend/plugins/gitlab/tasks/mr_convertor.go | 44 ++++++------- .../gitlab/tasks/mr_detail_extractor.go | 12 ++-- ...ment_convertor.go => mr_note_convertor.go} | 53 +++++++--------- 15 files changed, 234 insertions(+), 247 deletions(-) rename backend/plugins/gitlab/tasks/{mr_comment_convertor.go => mr_note_convertor.go} (71%) diff --git a/backend/helpers/pluginhelper/api/enrich_with_regex.go b/backend/helpers/pluginhelper/api/enrich_with_regex.go index 5e7bd2a3a76..4201571e274 100644 --- a/backend/helpers/pluginhelper/api/enrich_with_regex.go +++ b/backend/helpers/pluginhelper/api/enrich_with_regex.go @@ -105,3 +105,11 @@ func (r *RegexEnricher) ReturnNameIfOmittedOrMatched(name string, targets ...str } return r.ReturnNameIfMatched(name, targets...) } + +func (r *RegexEnricher) PlainMap() map[string]string { + m := make(map[string]string) + for k, v := range r.regexpMap { + m[k] = v.String() + } + return m +} diff --git a/backend/plugins/gitlab/tasks/deployment_convertor.go b/backend/plugins/gitlab/tasks/deployment_convertor.go index d7ff0d7741f..1df80ec543e 100644 --- a/backend/plugins/gitlab/tasks/deployment_convertor.go +++ b/backend/plugins/gitlab/tasks/deployment_convertor.go @@ -19,7 +19,6 @@ package tasks import ( "fmt" - "reflect" "time" "github.com/apache/incubator-devlake/core/dal" @@ -50,9 +49,9 @@ var ConvertDeploymentMeta = plugin.SubTaskMeta{ // ConvertDeployment should be split into two task theoretically // But in GitLab, all deployments have commits, so there is no need to change it. -func ConvertDeployment(taskCtx plugin.SubTaskContext) errors.Error { - rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_DEPLOYMENT) - db := taskCtx.GetDal() +func ConvertDeployment(subtaskCtx plugin.SubTaskContext) errors.Error { + subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_DEPLOYMENT) + db := subtaskCtx.GetDal() repo := &models.GitlabProject{} err := db.First(repo, dal.Where("gitlab_id = ? and connection_id = ?", data.Options.ProjectId, data.Options.ConnectionId)) @@ -61,25 +60,24 @@ func ConvertDeployment(taskCtx plugin.SubTaskContext) errors.Error { } projectIdGen := didgen.NewDomainIdGenerator(&models.GitlabProject{}) - - cursor, err := db.Cursor( - dal.From(&models.GitlabDeployment{}), - dal.Where("connection_id = ? AND gitlab_id = ?", data.Options.ConnectionId, data.Options.ProjectId), - ) - if err != nil { - return err - } - defer cursor.Close() - idGen := didgen.NewDomainIdGenerator(&models.GitlabDeployment{}) - converter, err := api.NewDataConverter(api.DataConverterArgs{ - InputRowType: reflect.TypeOf(models.GitlabDeployment{}), - Input: cursor, - RawDataSubTaskArgs: *rawDataSubTaskArgs, - Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { - gitlabDeployment := inputRow.(*models.GitlabDeployment) - + converter, err := api.NewStatefulDataConverter[models.GitlabDeployment](&api.StatefulDataConverterArgs[models.GitlabDeployment]{ + SubtaskCommonArgs: subtaskCommonArgs, + Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { + clauses := []dal.Clause{ + dal.From(&models.GitlabDeployment{}), + dal.Where("connection_id = ? AND gitlab_id = ?", data.Options.ConnectionId, data.Options.ProjectId), + } + if stateManager.IsIncremental() { + since := stateManager.GetSince() + if since != nil { + clauses = append(clauses, dal.Where("updated_at >= ? ", since)) + } + } + return db.Cursor(clauses...) + }, + Convert: func(gitlabDeployment *models.GitlabDeployment) ([]interface{}, errors.Error) { var duration *float64 if gitlabDeployment.DeployableDuration != nil { deployableDuration := cast.ToFloat64(*gitlabDeployment.DeployableDuration) diff --git a/backend/plugins/gitlab/tasks/deployment_extractor.go b/backend/plugins/gitlab/tasks/deployment_extractor.go index 0b439aceb56..274714b2df4 100644 --- a/backend/plugins/gitlab/tasks/deployment_extractor.go +++ b/backend/plugins/gitlab/tasks/deployment_extractor.go @@ -44,10 +44,10 @@ var ExtractDeploymentMeta = plugin.SubTaskMeta{ } func ExtractDeployment(taskCtx plugin.SubTaskContext) errors.Error { - rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_DEPLOYMENT) + subtaskCommonArgs, data := CreateSubtaskCommonArgs(taskCtx, RAW_DEPLOYMENT) - extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ - RawDataSubTaskArgs: *rawDataSubTaskArgs, + extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs{ + SubtaskCommonArgs: subtaskCommonArgs, Extract: func(row *api.RawData) ([]interface{}, errors.Error) { deploymentResp := &GitlabDeploymentResp{} err := errors.Convert(json.Unmarshal(row.Data, deploymentResp)) diff --git a/backend/plugins/gitlab/tasks/issue_assignee_convertor.go b/backend/plugins/gitlab/tasks/issue_assignee_convertor.go index f4a70065d8a..65c3abeb138 100644 --- a/backend/plugins/gitlab/tasks/issue_assignee_convertor.go +++ b/backend/plugins/gitlab/tasks/issue_assignee_convertor.go @@ -18,14 +18,12 @@ limitations under the License. package tasks import ( - "reflect" - "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" "github.com/apache/incubator-devlake/core/plugin" - helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/gitlab/models" ) @@ -43,32 +41,33 @@ var ConvertIssueAssigneeMeta = plugin.SubTaskMeta{ Dependencies: []*plugin.SubTaskMeta{&ExtractApiIssuesMeta}, } -func ConvertIssueAssignee(taskCtx plugin.SubTaskContext) errors.Error { - db := taskCtx.GetDal() - rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_ISSUE_TABLE) - - cursor, err := db.Cursor( - dal.From(&models.GitlabIssueAssignee{}), - dal.Where("project_id = ? and connection_id=?", data.Options.ProjectId, data.Options.ConnectionId), - ) - if err != nil { - return err - } - defer cursor.Close() +func ConvertIssueAssignee(subtaskCtx plugin.SubTaskContext) errors.Error { + subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_ISSUE_TABLE) + db := subtaskCtx.GetDal() issueIdGen := didgen.NewDomainIdGenerator(&models.GitlabIssue{}) accountIdGen := didgen.NewDomainIdGenerator(&models.GitlabAccount{}) - converter, err := helper.NewDataConverter(helper.DataConverterArgs{ - RawDataSubTaskArgs: *rawDataSubTaskArgs, - InputRowType: reflect.TypeOf(models.GitlabIssueAssignee{}), - Input: cursor, - Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { - input := inputRow.(*models.GitlabIssueAssignee) + converter, err := api.NewStatefulDataConverter[models.GitlabIssueAssignee](&api.StatefulDataConverterArgs[models.GitlabIssueAssignee]{ + SubtaskCommonArgs: subtaskCommonArgs, + Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { + clauses := []dal.Clause{ + dal.From(&models.GitlabIssueAssignee{}), + dal.Where("connection_id = ? AND project_id = ?", data.Options.ConnectionId, data.Options.ProjectId), + } + if stateManager.IsIncremental() { + since := stateManager.GetSince() + if since != nil { + clauses = append(clauses, dal.Where("updated_at >= ? ", since)) + } + } + return db.Cursor(clauses...) + }, + Convert: func(issueAssignee *models.GitlabIssueAssignee) ([]interface{}, errors.Error) { domainIssueAssignee := &ticket.IssueAssignee{ - IssueId: issueIdGen.Generate(data.Options.ConnectionId, input.GitlabId), - AssigneeId: accountIdGen.Generate(data.Options.ConnectionId, input.AssigneeId), - AssigneeName: input.AssigneeName, + IssueId: issueIdGen.Generate(data.Options.ConnectionId, issueAssignee.GitlabId), + AssigneeId: accountIdGen.Generate(data.Options.ConnectionId, issueAssignee.AssigneeId), + AssigneeName: issueAssignee.AssigneeName, } return []interface{}{domainIssueAssignee}, nil }, diff --git a/backend/plugins/gitlab/tasks/issue_convertor.go b/backend/plugins/gitlab/tasks/issue_convertor.go index 9be3c1d1822..5f120c52ec1 100644 --- a/backend/plugins/gitlab/tasks/issue_convertor.go +++ b/backend/plugins/gitlab/tasks/issue_convertor.go @@ -18,7 +18,6 @@ limitations under the License. package tasks import ( - "reflect" "strconv" "strings" @@ -28,7 +27,7 @@ import ( "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" "github.com/apache/incubator-devlake/core/plugin" - helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/gitlab/models" ) @@ -45,32 +44,31 @@ var ConvertIssuesMeta = plugin.SubTaskMeta{ Dependencies: []*plugin.SubTaskMeta{&ConvertApiMrCommitsMeta}, } -func ConvertIssues(taskCtx plugin.SubTaskContext) errors.Error { - db := taskCtx.GetDal() - rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_ISSUE_TABLE) - projectId := data.Options.ProjectId - - clauses := []dal.Clause{ - dal.Select("issues.*"), - dal.From("_tool_gitlab_issues issues"), - dal.Where("project_id = ? and connection_id = ?", projectId, data.Options.ConnectionId), - } - cursor, err := db.Cursor(clauses...) - if err != nil { - return err - } - defer cursor.Close() +func ConvertIssues(subtaskCtx plugin.SubTaskContext) errors.Error { + subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_ISSUE_TABLE) + db := subtaskCtx.GetDal() + projectId := data.Options.ProjectId issueIdGen := didgen.NewDomainIdGenerator(&models.GitlabIssue{}) accountIdGen := didgen.NewDomainIdGenerator(&models.GitlabAccount{}) boardIdGen := didgen.NewDomainIdGenerator(&models.GitlabProject{}) - converter, err := helper.NewDataConverter(helper.DataConverterArgs{ - RawDataSubTaskArgs: *rawDataSubTaskArgs, - InputRowType: reflect.TypeOf(models.GitlabIssue{}), - Input: cursor, - Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { - issue := inputRow.(*models.GitlabIssue) + converter, err := api.NewStatefulDataConverter[models.GitlabIssue](&api.StatefulDataConverterArgs[models.GitlabIssue]{ + SubtaskCommonArgs: subtaskCommonArgs, + Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { + clauses := []dal.Clause{ + dal.From(&models.GitlabIssue{}), + dal.Where("connection_id = ? AND gitlab_id = ?", data.Options.ConnectionId, data.Options.ProjectId), + } + if stateManager.IsIncremental() { + since := stateManager.GetSince() + if since != nil { + clauses = append(clauses, dal.Where("updated_at >= ? ", since)) + } + } + return db.Cursor(clauses...) + }, + Convert: func(issue *models.GitlabIssue) ([]interface{}, errors.Error) { domainIssue := &ticket.Issue{ DomainEntity: domainlayer.DomainEntity{Id: issueIdGen.Generate(data.Options.ConnectionId, issue.GitlabId)}, IssueKey: strconv.Itoa(issue.Number), diff --git a/backend/plugins/gitlab/tasks/issue_extractor.go b/backend/plugins/gitlab/tasks/issue_extractor.go index 717e0ca9861..9685339af74 100644 --- a/backend/plugins/gitlab/tasks/issue_extractor.go +++ b/backend/plugins/gitlab/tasks/issue_extractor.go @@ -134,7 +134,8 @@ type IssuesResponse struct { } func ExtractApiIssues(taskCtx plugin.SubTaskContext) errors.Error { - rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_ISSUE_TABLE) + subtaskCommonArgs, data := CreateSubtaskCommonArgs(taskCtx, RAW_ISSUE_TABLE) + config := data.Options.ScopeConfig var issueSeverityRegex *regexp.Regexp var issueComponentRegex *regexp.Regexp @@ -162,8 +163,13 @@ func ExtractApiIssues(taskCtx plugin.SubTaskContext) errors.Error { return errors.Default.Wrap(err, "regexp Compile issuePriority failed") } } - extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ - RawDataSubTaskArgs: *rawDataSubTaskArgs, + subtaskCommonArgs.SubtaskConfig = map[string]interface{}{ + "issueSeverity": issueSeverity, + "issueComponent": issueComponent, + "issuePriorityRegex": issuePriorityRegex, + } + extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs{ + SubtaskCommonArgs: subtaskCommonArgs, Extract: func(row *api.RawData) ([]interface{}, errors.Error) { body := &IssuesResponse{} err := errors.Convert(json.Unmarshal(row.Data, body)) diff --git a/backend/plugins/gitlab/tasks/issue_label_convertor.go b/backend/plugins/gitlab/tasks/issue_label_convertor.go index c5e8968dda6..4ffa1cc2f22 100644 --- a/backend/plugins/gitlab/tasks/issue_label_convertor.go +++ b/backend/plugins/gitlab/tasks/issue_label_convertor.go @@ -18,14 +18,12 @@ limitations under the License. package tasks import ( - "reflect" - "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" "github.com/apache/incubator-devlake/core/plugin" - helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/gitlab/models" ) @@ -42,20 +40,11 @@ var ConvertIssueLabelsMeta = plugin.SubTaskMeta{ Dependencies: []*plugin.SubTaskMeta{&ConvertIssuesMeta}, } -func ConvertIssueLabels(taskCtx plugin.SubTaskContext) errors.Error { - db := taskCtx.GetDal() - rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_ISSUE_TABLE) +func ConvertIssueLabels(subtaskCtx plugin.SubTaskContext) errors.Error { + subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_ISSUE_TABLE) + db := subtaskCtx.GetDal() projectId := data.Options.ProjectId - clauses := []dal.Clause{ - dal.Select("*"), - dal.From(&models.GitlabIssueLabel{}), - dal.Join(`left join _tool_gitlab_issues on - _tool_gitlab_issues.gitlab_id = _tool_gitlab_issue_labels.issue_id`), - dal.Where(`_tool_gitlab_issues.project_id = ? - and _tool_gitlab_issues.connection_id = ?`, - projectId, data.Options.ConnectionId), - dal.Orderby("issue_id ASC"), - } + clauses := []dal.Clause{} cursor, err := db.Cursor(clauses...) if err != nil { @@ -65,12 +54,28 @@ func ConvertIssueLabels(taskCtx plugin.SubTaskContext) errors.Error { issueIdGen := didgen.NewDomainIdGenerator(&models.GitlabIssue{}) - converter, err := helper.NewDataConverter(helper.DataConverterArgs{ - RawDataSubTaskArgs: *rawDataSubTaskArgs, - InputRowType: reflect.TypeOf(models.GitlabIssueLabel{}), - Input: cursor, - Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { - issueLabel := inputRow.(*models.GitlabIssueLabel) + converter, err := api.NewStatefulDataConverter[models.GitlabIssueLabel](&api.StatefulDataConverterArgs[models.GitlabIssueLabel]{ + SubtaskCommonArgs: subtaskCommonArgs, + Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { + clauses := []dal.Clause{ + dal.Select("*"), + dal.From(&models.GitlabIssueLabel{}), + dal.Join(`left join _tool_gitlab_issues on + _tool_gitlab_issues.gitlab_id = _tool_gitlab_issue_labels.issue_id`), + dal.Where(`_tool_gitlab_issues.project_id = ? + and _tool_gitlab_issues.connection_id = ?`, + projectId, data.Options.ConnectionId), + dal.Orderby("issue_id ASC"), + } + if stateManager.IsIncremental() { + since := stateManager.GetSince() + if since != nil { + clauses = append(clauses, dal.Where("_tool_gitlab_issues.updated_at >= ? ", since)) + } + } + return db.Cursor(clauses...) + }, + Convert: func(issueLabel *models.GitlabIssueLabel) ([]interface{}, errors.Error) { domainIssueLabel := &ticket.IssueLabel{ IssueId: issueIdGen.Generate(data.Options.ConnectionId, issueLabel.IssueId), LabelName: issueLabel.LabelName, diff --git a/backend/plugins/gitlab/tasks/job_convertor.go b/backend/plugins/gitlab/tasks/job_convertor.go index 58bd8f712e4..9e1042ac310 100644 --- a/backend/plugins/gitlab/tasks/job_convertor.go +++ b/backend/plugins/gitlab/tasks/job_convertor.go @@ -18,7 +18,6 @@ limitations under the License. package tasks import ( - "reflect" "time" "github.com/apache/incubator-devlake/core/dal" @@ -28,6 +27,7 @@ import ( "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/gitlab/models" gitlabModels "github.com/apache/incubator-devlake/plugins/gitlab/models" ) @@ -45,34 +45,30 @@ var ConvertJobMeta = plugin.SubTaskMeta{ } func ConvertJobs(taskCtx plugin.SubTaskContext) (err errors.Error) { + subtaskCommonArgs, data := CreateSubtaskCommonArgs(taskCtx, RAW_JOB_TABLE) db := taskCtx.GetDal() - data := taskCtx.GetData().(*GitlabTaskData) regexEnricher := data.RegexEnricher - - cursor, err := db.Cursor(dal.From(gitlabModels.GitlabJob{}), - dal.Where("project_id = ? and connection_id = ?", data.Options.ProjectId, data.Options.ConnectionId)) - if err != nil { - return err - } - defer cursor.Close() + subtaskCommonArgs.SubtaskConfig = regexEnricher.PlainMap() jobIdGen := didgen.NewDomainIdGenerator(&gitlabModels.GitlabJob{}) projectIdGen := didgen.NewDomainIdGenerator(&gitlabModels.GitlabProject{}) pipelineIdGen := didgen.NewDomainIdGenerator(&gitlabModels.GitlabPipeline{}) - converter, err := api.NewDataConverter(api.DataConverterArgs{ - InputRowType: reflect.TypeOf(gitlabModels.GitlabJob{}), - Input: cursor, - RawDataSubTaskArgs: api.RawDataSubTaskArgs{ - Ctx: taskCtx, - Params: gitlabModels.GitlabApiParams{ - ConnectionId: data.Options.ConnectionId, - ProjectId: data.Options.ProjectId, - }, - Table: RAW_JOB_TABLE, + converter, err := api.NewStatefulDataConverter[models.GitlabJob](&api.StatefulDataConverterArgs[models.GitlabJob]{ + SubtaskCommonArgs: subtaskCommonArgs, + Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { + clauses := []dal.Clause{ + dal.From(gitlabModels.GitlabJob{}), + dal.Where("project_id = ? and connection_id = ?", data.Options.ProjectId, data.Options.ConnectionId), + } + if stateManager.IsIncremental() { + since := stateManager.GetSince() + if since != nil { + clauses = append(clauses, dal.Where("updated_at >= ? ", since)) + } + } + return db.Cursor(clauses...) }, - Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { - gitlabJob := inputRow.(*gitlabModels.GitlabJob) - + Convert: func(gitlabJob *models.GitlabJob) ([]interface{}, errors.Error) { createdAt := time.Now() if gitlabJob.GitlabCreatedAt != nil { createdAt = *gitlabJob.GitlabCreatedAt diff --git a/backend/plugins/gitlab/tasks/job_extractor.go b/backend/plugins/gitlab/tasks/job_extractor.go index ff5031ce6d5..ec4cd5427c5 100644 --- a/backend/plugins/gitlab/tasks/job_extractor.go +++ b/backend/plugins/gitlab/tasks/job_extractor.go @@ -60,10 +60,10 @@ var ExtractApiJobsMeta = plugin.SubTaskMeta{ } func ExtractApiJobs(taskCtx plugin.SubTaskContext) errors.Error { - rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_JOB_TABLE) + subtaskCommonArgs, data := CreateSubtaskCommonArgs(taskCtx, RAW_JOB_TABLE) - extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ - RawDataSubTaskArgs: *rawDataSubTaskArgs, + extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs{ + SubtaskCommonArgs: subtaskCommonArgs, Extract: func(row *api.RawData) ([]interface{}, errors.Error) { // create gitlab commit gitlabApiJob := &ApiJob{} diff --git a/backend/plugins/gitlab/tasks/mr_assignee_convertor.go b/backend/plugins/gitlab/tasks/mr_assignee_convertor.go index d638d1dc0b3..541634748cf 100644 --- a/backend/plugins/gitlab/tasks/mr_assignee_convertor.go +++ b/backend/plugins/gitlab/tasks/mr_assignee_convertor.go @@ -18,14 +18,12 @@ limitations under the License. package tasks import ( - "reflect" - "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer/code" "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" "github.com/apache/incubator-devlake/core/plugin" - helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/gitlab/models" ) @@ -42,36 +40,30 @@ var ConvertMrAssigneesMeta = plugin.SubTaskMeta{ Dependencies: []*plugin.SubTaskMeta{&ExtractApiMergeRequestDetailsMeta}, } -func ConvertMrAssignees(taskCtx plugin.SubTaskContext) errors.Error { - db := taskCtx.GetDal() - rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_MERGE_REQUEST_TABLE) +func ConvertMrAssignees(subtaskCtx plugin.SubTaskContext) errors.Error { + subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_MERGE_REQUEST_TABLE) + db := subtaskCtx.GetDal() projectId := data.Options.ProjectId - clauses := []dal.Clause{ - dal.Select("_tool_gitlab_assignees.*"), - dal.From(&models.GitlabAssignee{}), - dal.Join(`left join _tool_gitlab_merge_requests mr on - mr.gitlab_id = _tool_gitlab_assignees.merge_request_id`), - dal.Where(`mr.project_id = ? - and mr.connection_id = ?`, - projectId, data.Options.ConnectionId), - dal.Orderby("_tool_gitlab_assignees.merge_request_id ASC"), - } - - cursor, err := db.Cursor(clauses...) - if err != nil { - return err - } - defer cursor.Close() - mrIdGen := didgen.NewDomainIdGenerator(&models.GitlabMergeRequest{}) accountIdGen := didgen.NewDomainIdGenerator(&models.GitlabAccount{}) - converter, err := helper.NewDataConverter(helper.DataConverterArgs{ - RawDataSubTaskArgs: *rawDataSubTaskArgs, - InputRowType: reflect.TypeOf(models.GitlabAssignee{}), - Input: cursor, - Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { - mrAssignee := inputRow.(*models.GitlabAssignee) + converter, err := api.NewStatefulDataConverter[models.GitlabAssignee](&api.StatefulDataConverterArgs[models.GitlabAssignee]{ + SubtaskCommonArgs: subtaskCommonArgs, + Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { + clauses := []dal.Clause{ + dal.From(&models.GitlabAssignee{}), + dal.Where(`project_id = ? and connection_id = ?`, projectId, data.Options.ConnectionId), + dal.Orderby("merge_request_id ASC"), + } + if stateManager.IsIncremental() { + since := stateManager.GetSince() + if since != nil { + clauses = append(clauses, dal.Where("updated_at >= ? ", since)) + } + } + return db.Cursor(clauses...) + }, + Convert: func(mrAssignee *models.GitlabAssignee) ([]interface{}, errors.Error) { domainPrAssigne := &code.PullRequestAssignee{ PullRequestId: mrIdGen.Generate(data.Options.ConnectionId, mrAssignee.MergeRequestId), AssigneeId: accountIdGen.Generate(data.Options.ConnectionId, mrAssignee.AssigneeId), diff --git a/backend/plugins/gitlab/tasks/mr_commit_convertor.go b/backend/plugins/gitlab/tasks/mr_commit_convertor.go index 74ae0d926bb..91ed65c4238 100644 --- a/backend/plugins/gitlab/tasks/mr_commit_convertor.go +++ b/backend/plugins/gitlab/tasks/mr_commit_convertor.go @@ -18,14 +18,12 @@ limitations under the License. package tasks import ( - "reflect" - "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer/code" "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" "github.com/apache/incubator-devlake/core/plugin" - helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/gitlab/models" ) @@ -42,42 +40,36 @@ var ConvertApiMrCommitsMeta = plugin.SubTaskMeta{ Dependencies: []*plugin.SubTaskMeta{&ConvertMrCommentMeta}, } -func ConvertApiMergeRequestsCommits(taskCtx plugin.SubTaskContext) errors.Error { - rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_MERGE_REQUEST_COMMITS_TABLE) - db := taskCtx.GetDal() - - clauses := []dal.Clause{ - dal.From(&models.GitlabMrCommit{}), - dal.Join(`left join _tool_gitlab_merge_requests - on _tool_gitlab_merge_requests.gitlab_id = - _tool_gitlab_mr_commits.merge_request_id`), - dal.Where(`_tool_gitlab_merge_requests.project_id = ? - and _tool_gitlab_merge_requests.connection_id = ?`, - data.Options.ProjectId, data.Options.ConnectionId), - dal.Orderby("merge_request_id ASC"), - } +func ConvertApiMergeRequestsCommits(subtaskCtx plugin.SubTaskContext) errors.Error { + subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_MERGE_REQUEST_COMMITS_TABLE) + db := subtaskCtx.GetDal() - cursor, err := db.Cursor(clauses...) - if err != nil { - return err - } - - // TODO: adopt batch indate operation domainIdGenerator := didgen.NewDomainIdGenerator(&models.GitlabMergeRequest{}) - converter, err := helper.NewDataConverter(helper.DataConverterArgs{ - RawDataSubTaskArgs: *rawDataSubTaskArgs, - InputRowType: reflect.TypeOf(models.GitlabMrCommit{}), - Input: cursor, - - Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { - GitlabMrCommit := inputRow.(*models.GitlabMrCommit) + converter, err := api.NewStatefulDataConverter[models.GitlabMrCommit](&api.StatefulDataConverterArgs[models.GitlabMrCommit]{ + SubtaskCommonArgs: subtaskCommonArgs, + Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { + clauses := []dal.Clause{ + dal.Select("c.*"), + dal.From("_tool_gitlab_mr_commits c"), + dal.Join(`LEFT JOIN _tool_gitlab_merge_requests mr ON mr.gitlab_id = c.merge_request_id`), + dal.Where(`mr.project_id = ? and mr.connection_id = ?`, data.Options.ProjectId, data.Options.ConnectionId), + } + if stateManager.IsIncremental() { + since := stateManager.GetSince() + if since != nil { + clauses = append(clauses, dal.Where("c.updated_at >= ? ", since)) + } + } + return db.Cursor(clauses...) + }, + Convert: func(mrCommit *models.GitlabMrCommit) ([]interface{}, errors.Error) { domainPrcommit := &code.PullRequestCommit{ - CommitSha: GitlabMrCommit.CommitSha, - PullRequestId: domainIdGenerator.Generate(data.Options.ConnectionId, GitlabMrCommit.MergeRequestId), - CommitAuthorName: GitlabMrCommit.CommitAuthorName, - CommitAuthorEmail: GitlabMrCommit.CommitAuthorEmail, - CommitAuthoredDate: *GitlabMrCommit.CommitAuthoredDate, + CommitSha: mrCommit.CommitSha, + PullRequestId: domainIdGenerator.Generate(data.Options.ConnectionId, mrCommit.MergeRequestId), + CommitAuthorName: mrCommit.CommitAuthorName, + CommitAuthorEmail: mrCommit.CommitAuthorEmail, + CommitAuthoredDate: *mrCommit.CommitAuthoredDate, } return []interface{}{ domainPrcommit, diff --git a/backend/plugins/gitlab/tasks/mr_commit_extractor.go b/backend/plugins/gitlab/tasks/mr_commit_extractor.go index 5c73c8665bc..12f0d96be98 100644 --- a/backend/plugins/gitlab/tasks/mr_commit_extractor.go +++ b/backend/plugins/gitlab/tasks/mr_commit_extractor.go @@ -40,11 +40,11 @@ var ExtractApiMrCommitsMeta = plugin.SubTaskMeta{ Dependencies: []*plugin.SubTaskMeta{&CollectApiMrCommitsMeta}, } -func ExtractApiMergeRequestsCommits(taskCtx plugin.SubTaskContext) errors.Error { - rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_MERGE_REQUEST_COMMITS_TABLE) +func ExtractApiMergeRequestsCommits(subtaskCtx plugin.SubTaskContext) errors.Error { + subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_MERGE_REQUEST_COMMITS_TABLE) - extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ - RawDataSubTaskArgs: *rawDataSubTaskArgs, + extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs{ + SubtaskCommonArgs: subtaskCommonArgs, Extract: func(row *api.RawData) ([]interface{}, errors.Error) { // create gitlab commit gitlabApiCommit := &GitlabApiCommit{} diff --git a/backend/plugins/gitlab/tasks/mr_convertor.go b/backend/plugins/gitlab/tasks/mr_convertor.go index 478276c71f4..b6b11e498d7 100644 --- a/backend/plugins/gitlab/tasks/mr_convertor.go +++ b/backend/plugins/gitlab/tasks/mr_convertor.go @@ -18,15 +18,13 @@ limitations under the License. package tasks import ( - "reflect" - "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer" "github.com/apache/incubator-devlake/core/models/domainlayer/code" "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" "github.com/apache/incubator-devlake/core/plugin" - helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/gitlab/models" ) @@ -43,32 +41,30 @@ var ConvertApiMergeRequestsMeta = plugin.SubTaskMeta{ Dependencies: []*plugin.SubTaskMeta{&ConvertProjectMeta}, } -func ConvertApiMergeRequests(taskCtx plugin.SubTaskContext) errors.Error { - rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_MERGE_REQUEST_TABLE) - db := taskCtx.GetDal() - clauses := []dal.Clause{ - dal.From(&models.GitlabMergeRequest{}), - dal.Where("project_id=? and connection_id = ?", data.Options.ProjectId, data.Options.ConnectionId), - } - - cursor, err := db.Cursor(clauses...) - if err != nil { - return err - } - defer cursor.Close() +func ConvertApiMergeRequests(subtaskCtx plugin.SubTaskContext) errors.Error { + subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_MERGE_REQUEST_TABLE) + db := subtaskCtx.GetDal() domainMrIdGenerator := didgen.NewDomainIdGenerator(&models.GitlabMergeRequest{}) domainRepoIdGenerator := didgen.NewDomainIdGenerator(&models.GitlabProject{}) domainUserIdGen := didgen.NewDomainIdGenerator(&models.GitlabAccount{}) - converter, err := helper.NewDataConverter(helper.DataConverterArgs{ - RawDataSubTaskArgs: *rawDataSubTaskArgs, - InputRowType: reflect.TypeOf(models.GitlabMergeRequest{}), - Input: cursor, - - Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { - gitlabMr := inputRow.(*models.GitlabMergeRequest) - + converter, err := api.NewStatefulDataConverter[models.GitlabMergeRequest](&api.StatefulDataConverterArgs[models.GitlabMergeRequest]{ + SubtaskCommonArgs: subtaskCommonArgs, + Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { + clauses := []dal.Clause{ + dal.From(&models.GitlabMergeRequest{}), + dal.Where("project_id=? and connection_id = ?", data.Options.ProjectId, data.Options.ConnectionId), + } + if stateManager.IsIncremental() { + since := stateManager.GetSince() + if since != nil { + clauses = append(clauses, dal.Where("updated_at >= ? ", since)) + } + } + return db.Cursor(clauses...) + }, + Convert: func(gitlabMr *models.GitlabMergeRequest) ([]interface{}, errors.Error) { domainPr := &code.PullRequest{ DomainEntity: domainlayer.DomainEntity{ Id: domainMrIdGenerator.Generate(data.Options.ConnectionId, gitlabMr.GitlabId), diff --git a/backend/plugins/gitlab/tasks/mr_detail_extractor.go b/backend/plugins/gitlab/tasks/mr_detail_extractor.go index 3dbc72e8af7..142cf2de875 100644 --- a/backend/plugins/gitlab/tasks/mr_detail_extractor.go +++ b/backend/plugins/gitlab/tasks/mr_detail_extractor.go @@ -40,8 +40,8 @@ var ExtractApiMergeRequestDetailsMeta = plugin.SubTaskMeta{ Dependencies: []*plugin.SubTaskMeta{&ExtractApiCommitsMeta}, } -func ExtractApiMergeRequestDetails(taskCtx plugin.SubTaskContext) errors.Error { - rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_MERGE_REQUEST_DETAIL_TABLE) +func ExtractApiMergeRequestDetails(subtaskCtx plugin.SubTaskContext) errors.Error { + subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_MERGE_REQUEST_DETAIL_TABLE) config := data.Options.ScopeConfig var labelTypeRegex *regexp.Regexp var labelComponentRegex *regexp.Regexp @@ -60,8 +60,12 @@ func ExtractApiMergeRequestDetails(taskCtx plugin.SubTaskContext) errors.Error { return errors.Default.Wrap(err, "regexp Compile prComponent failed") } } - extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ - RawDataSubTaskArgs: *rawDataSubTaskArgs, + subtaskCommonArgs.SubtaskConfig = map[string]string{ + "prType": prType, + "prComponent": prComponent, + } + extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs{ + SubtaskCommonArgs: subtaskCommonArgs, Extract: func(row *api.RawData) ([]interface{}, errors.Error) { mr := &MergeRequestRes{} err := errors.Convert(json.Unmarshal(row.Data, mr)) diff --git a/backend/plugins/gitlab/tasks/mr_comment_convertor.go b/backend/plugins/gitlab/tasks/mr_note_convertor.go similarity index 71% rename from backend/plugins/gitlab/tasks/mr_comment_convertor.go rename to backend/plugins/gitlab/tasks/mr_note_convertor.go index 3b6d1cceef0..27005e15eff 100644 --- a/backend/plugins/gitlab/tasks/mr_comment_convertor.go +++ b/backend/plugins/gitlab/tasks/mr_note_convertor.go @@ -18,15 +18,13 @@ limitations under the License. package tasks import ( - "reflect" - "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer" "github.com/apache/incubator-devlake/core/models/domainlayer/code" "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" "github.com/apache/incubator-devlake/core/plugin" - helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/gitlab/models" ) @@ -36,44 +34,39 @@ func init() { var ConvertMrCommentMeta = plugin.SubTaskMeta{ Name: "Convert MR Comments", - EntryPoint: ConvertMergeRequestComment, + EntryPoint: ConvertMergeRequestNote, EnabledByDefault: true, Description: "Add domain layer Comment according to GitlabMrComment", DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW}, Dependencies: []*plugin.SubTaskMeta{&ConvertApiMergeRequestsMeta}, } -func ConvertMergeRequestComment(taskCtx plugin.SubTaskContext) errors.Error { - rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_MERGE_REQUEST_NOTES_TABLE) - db := taskCtx.GetDal() - clauses := []dal.Clause{ - dal.From(&models.GitlabMrComment{}), - dal.Join(`left join _tool_gitlab_merge_requests on - _tool_gitlab_merge_requests.gitlab_id = - _tool_gitlab_mr_comments.merge_request_id`), - dal.Where(`_tool_gitlab_merge_requests.project_id = ? - and _tool_gitlab_mr_comments.connection_id = ?`, - data.Options.ProjectId, data.Options.ConnectionId), - } - - cursor, err := db.Cursor(clauses...) - if err != nil { - return err - } - defer cursor.Close() +func ConvertMergeRequestNote(subtaskCtx plugin.SubTaskContext) errors.Error { + subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_MERGE_REQUEST_NOTES_TABLE) + db := subtaskCtx.GetDal() domainIdGeneratorComment := didgen.NewDomainIdGenerator(&models.GitlabMrComment{}) prIdGen := didgen.NewDomainIdGenerator(&models.GitlabMergeRequest{}) accountIdGen := didgen.NewDomainIdGenerator(&models.GitlabAccount{}) - converter, err := helper.NewDataConverter(helper.DataConverterArgs{ - RawDataSubTaskArgs: *rawDataSubTaskArgs, - InputRowType: reflect.TypeOf(models.GitlabMrComment{}), - Input: cursor, - - Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { - gitlabComments := inputRow.(*models.GitlabMrComment) - + converter, err := api.NewStatefulDataConverter[models.GitlabMrComment](&api.StatefulDataConverterArgs[models.GitlabMrComment]{ + SubtaskCommonArgs: subtaskCommonArgs, + Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { + clauses := []dal.Clause{ + dal.Select("c.*"), + dal.From("_tool_gitlab_mr_comments c"), + dal.Join(`LEFT JOIN _tool_gitlab_merge_requests mr ON mr.gitlab_id = c.merge_request_id`), + dal.Where(`mr.project_id = ? and mr.connection_id = ?`, data.Options.ProjectId, data.Options.ConnectionId), + } + if stateManager.IsIncremental() { + since := stateManager.GetSince() + if since != nil { + clauses = append(clauses, dal.Where("c.updated_at >= ? ", since)) + } + } + return db.Cursor(clauses...) + }, + Convert: func(gitlabComments *models.GitlabMrComment) ([]interface{}, errors.Error) { domainComment := &code.PullRequestComment{ DomainEntity: domainlayer.DomainEntity{ Id: domainIdGeneratorComment.Generate(data.Options.ConnectionId, gitlabComments.GitlabId), From dbd6c1c3009b7120361ffa3a84d5057d36c5d60e Mon Sep 17 00:00:00 2001 From: Klesh Wong Date: Wed, 4 Sep 2024 16:06:32 +0800 Subject: [PATCH 03/23] feat: removed mr_enricher and all gitlab transformers support incr-mode --- .../20240904_remove_mr_review_fields.go | 40 +++++ backend/plugins/gitlab/models/mr.go | 4 +- backend/plugins/gitlab/tasks/mr_enricher.go | 165 ------------------ .../plugins/gitlab/tasks/mr_enricher_test.go | 112 ------------ .../gitlab/tasks/mr_label_convertor.go | 51 +++--- .../plugins/gitlab/tasks/mr_note_extractor.go | 8 +- .../gitlab/tasks/mr_reviewer_convertor.go | 51 +++--- .../gitlab/tasks/pipeline_commit_convertor.go | 45 ++--- .../gitlab/tasks/pipeline_detail_convertor.go | 43 ++--- .../gitlab/tasks/pipeline_detail_extractor.go | 8 +- .../gitlab/tasks/pipeline_extractor.go | 8 +- .../gitlab/tasks/trigger_job_extractor.go | 8 +- 12 files changed, 139 insertions(+), 404 deletions(-) create mode 100644 backend/plugins/gitlab/models/migrationscripts/20240904_remove_mr_review_fields.go delete mode 100644 backend/plugins/gitlab/tasks/mr_enricher.go delete mode 100644 backend/plugins/gitlab/tasks/mr_enricher_test.go diff --git a/backend/plugins/gitlab/models/migrationscripts/20240904_remove_mr_review_fields.go b/backend/plugins/gitlab/models/migrationscripts/20240904_remove_mr_review_fields.go new file mode 100644 index 00000000000..2628033f06f --- /dev/null +++ b/backend/plugins/gitlab/models/migrationscripts/20240904_remove_mr_review_fields.go @@ -0,0 +1,40 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +var _ plugin.MigrationScript = (*removeMrEnricherFields)(nil) + +type removeMrEnricherFields struct{} + +func (script *removeMrEnricherFields) Up(basicRes context.BasicRes) errors.Error { + return basicRes.GetDal().DropColumns("_tool_gitlab_merge_requests", "first_comment_time", "review_rounds") +} + +func (*removeMrEnricherFields) Version() uint64 { + return 20240904152414 +} + +func (*removeMrEnricherFields) Name() string { + return "remove mr review fields in favor of dora plugin" +} diff --git a/backend/plugins/gitlab/models/mr.go b/backend/plugins/gitlab/models/mr.go index 6ac3c9b7690..c25df4ba07f 100644 --- a/backend/plugins/gitlab/models/mr.go +++ b/backend/plugins/gitlab/models/mr.go @@ -50,9 +50,7 @@ type GitlabMergeRequest struct { Description string AuthorUsername string `gorm:"type:varchar(255)"` AuthorUserId int - Component string `gorm:"type:varchar(255)"` - FirstCommentTime *time.Time `gorm:"comment:Time when the first comment occurred"` - ReviewRounds int `gorm:"comment:How many rounds of review this MR went through"` + Component string `gorm:"type:varchar(255)"` common.NoPKModel } diff --git a/backend/plugins/gitlab/tasks/mr_enricher.go b/backend/plugins/gitlab/tasks/mr_enricher.go deleted file mode 100644 index 327765208bc..00000000000 --- a/backend/plugins/gitlab/tasks/mr_enricher.go +++ /dev/null @@ -1,165 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package tasks - -import ( - "reflect" - "time" - - "github.com/apache/incubator-devlake/core/dal" - "github.com/apache/incubator-devlake/core/errors" - "github.com/apache/incubator-devlake/core/plugin" - helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" - "github.com/apache/incubator-devlake/plugins/gitlab/models" -) - -func init() { - RegisterSubtaskMeta(&EnrichMergeRequestsMeta) -} - -var EnrichMergeRequestsMeta = plugin.SubTaskMeta{ - Name: "Enrich Merge Requests", - EntryPoint: EnrichMergeRequests, - EnabledByDefault: true, - Description: "Enrich merge requests data from GitlabCommit, GitlabMrNote and GitlabMergeRequest", - DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW}, - Dependencies: []*plugin.SubTaskMeta{&ExtractApiJobsMeta}, -} - -func EnrichMergeRequests(taskCtx plugin.SubTaskContext) errors.Error { - rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_MERGE_REQUEST_TABLE) - - db := taskCtx.GetDal() - clauses := []dal.Clause{ - dal.From(&models.GitlabMergeRequest{}), - dal.Where("project_id=? and connection_id = ?", data.Options.ProjectId, data.Options.ConnectionId), - } - - cursor, err := db.Cursor(clauses...) - if err != nil { - return err - } // get mrs from theDB - defer cursor.Close() - - converter, err := helper.NewDataConverter(helper.DataConverterArgs{ - RawDataSubTaskArgs: *rawDataSubTaskArgs, - InputRowType: reflect.TypeOf(models.GitlabMergeRequest{}), - Input: cursor, - - Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { - gitlabMr := inputRow.(*models.GitlabMergeRequest) - // enrich first_comment_time field - notes := make([]models.GitlabMrNote, 0) - // `system` = 0 is needed since we only care about human comments - noteClauses := []dal.Clause{ - dal.From(&models.GitlabMrNote{}), - dal.Where("merge_request_id = ? AND is_system = ? AND connection_id = ? ", - gitlabMr.GitlabId, false, data.Options.ConnectionId), - dal.Orderby("gitlab_created_at asc"), - } - err = db.All(¬es, noteClauses...) - if err != nil { - return nil, err - } - - commits := make([]models.GitlabCommit, 0) - commitClauses := []dal.Clause{ - dal.From(&models.GitlabCommit{}), - dal.Join(`join _tool_gitlab_mr_commits gmrc - on gmrc.commit_sha = _tool_gitlab_commits.sha`), - dal.Where("merge_request_id = ? AND gmrc.connection_id = ?", - gitlabMr.GitlabId, data.Options.ConnectionId), - dal.Orderby("authored_date asc"), - } - err = db.All(&commits, commitClauses...) - if err != nil { - return nil, err - } - - // calculate reviewRounds from commits and notes - reviewRounds := getReviewRounds(commits, notes) - gitlabMr.ReviewRounds = reviewRounds - - if len(notes) > 0 { - earliestNote, err := findEarliestNote(notes) - if err != nil { - return nil, err - } - if earliestNote != nil { - gitlabMr.FirstCommentTime = &earliestNote.GitlabCreatedAt - } - } - return []interface{}{ - gitlabMr, - }, nil - }, - }) - if err != nil { - return err - } - - return converter.Execute() -} - -func findEarliestNote(notes []models.GitlabMrNote) (*models.GitlabMrNote, errors.Error) { - var earliestNote *models.GitlabMrNote - earliestTime := time.Now() - for i := range notes { - if !notes[i].Resolvable { - continue - } - noteTime := notes[i].GitlabCreatedAt - if noteTime.Before(earliestTime) { - earliestTime = noteTime - earliestNote = ¬es[i] - } - } - return earliestNote, nil -} - -func getReviewRounds(commits []models.GitlabCommit, notes []models.GitlabMrNote) int { - i := 0 - j := 0 - reviewRounds := 0 - if len(commits) == 0 && len(notes) == 0 { - return 1 - } - // state is used to keep track of previous activity - // 0: init, 1: commit, 2: comment - // whenever state is switched to comment, we increment reviewRounds by 1 - state := 0 // 0, 1, 2 - for i < len(commits) && j < len(notes) { - if commits[i].AuthoredDate.Before(notes[j].GitlabCreatedAt) { - i++ - state = 1 - } else { - j++ - if state != 2 { - reviewRounds++ - } - state = 2 - } - } - // There's another implicit round of review in 2 scenarios - // One: the last state is commit (state == 1) - // Two: the last state is comment but there're still commits left - if state == 1 || i < len(commits) { - reviewRounds++ - } - return reviewRounds -} diff --git a/backend/plugins/gitlab/tasks/mr_enricher_test.go b/backend/plugins/gitlab/tasks/mr_enricher_test.go deleted file mode 100644 index f22fd275af4..00000000000 --- a/backend/plugins/gitlab/tasks/mr_enricher_test.go +++ /dev/null @@ -1,112 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package tasks - -import ( - "github.com/apache/incubator-devlake/plugins/gitlab/models" - "github.com/stretchr/testify/assert" - "testing" - "time" -) - -func TestFindEarliestNote(t *testing.T) { - baseTime, err := time.Parse(time.RFC3339, "2022-01-02T15:04:05Z") - assert.Nil(t, err) - // Create some sample notes - note1 := models.GitlabMrNote{Resolvable: true, GitlabCreatedAt: baseTime.Add(-time.Hour)} - note2 := models.GitlabMrNote{Resolvable: false, GitlabCreatedAt: baseTime.Add(-time.Minute)} - note3 := models.GitlabMrNote{Resolvable: true, GitlabCreatedAt: baseTime.Add(-time.Second)} - - // Call the function with the sample notes - notes := []models.GitlabMrNote{note1, note2, note3} - earliestNote, err := findEarliestNote(notes) - - // Check that no error was returned - if err != nil { - t.Errorf("findEarliestNote returned an error: %v", err) - } - - // Check that the correct note was returned - if earliestNote == nil { - t.Errorf("findEarliestNote returned nil, expected a note") - } - if !assert.Equal(t, note1, *earliestNote) { - t.Errorf("findEarliestNote returned the wrong note: got %v, expected %v", earliestNote, ¬e1) - } - - // Modify one of the notes to make it unresolvable - notes[0].Resolvable = false - - // Call the function again with the modified notes - earliestNote, err = findEarliestNote(notes) - - // Check that no error was returned - if err != nil { - t.Errorf("findEarliestNote returned an error: %v", err) - } - - // Check that the correct note was returned - if earliestNote == nil { - t.Errorf("findEarliestNote returned nil, expected a note") - } - if !assert.Equal(t, note3, *earliestNote) { - t.Errorf("findEarliestNote returned the wrong note: got %v, expected %v", earliestNote, ¬e3) - } -} - -func TestGetReviewRounds(t *testing.T) { - baseTime, err := time.Parse(time.RFC3339, "2022-01-02T15:04:05Z") - assert.Nil(t, err) - // Test case 1: empty input - var commits []models.GitlabCommit - var notes []models.GitlabMrNote - expected := 1 - if got := getReviewRounds(commits, notes); got != expected { - t.Errorf("getReviewRounds(%v, %v) = %d, expected %d", commits, notes, got, expected) - } - - // Test case 2: single comment - commits = []models.GitlabCommit{ - {AuthoredDate: baseTime.Add(-time.Hour * 2)}, - {AuthoredDate: baseTime.Add(-time.Hour)}, - } - notes = []models.GitlabMrNote{ - {GitlabCreatedAt: baseTime}, - } - expected = 1 - if got := getReviewRounds(commits, notes); got != expected { - t.Errorf("getReviewRounds(%v, %v) = %d, expected %d", commits, notes, got, expected) - } - - // Test case 3: multiple comments - commits = []models.GitlabCommit{ - {AuthoredDate: baseTime.Add(-time.Hour * 15)}, - {AuthoredDate: baseTime.Add(-time.Hour * 9)}, - {AuthoredDate: baseTime.Add(-time.Hour * 3)}, - {AuthoredDate: baseTime}, - } - notes = []models.GitlabMrNote{ - {GitlabCreatedAt: baseTime.Add(-time.Hour * 14)}, - {GitlabCreatedAt: baseTime.Add(-time.Hour * 7)}, - {GitlabCreatedAt: baseTime.Add(-time.Hour * 2)}, - } - expected = 4 - if got := getReviewRounds(commits, notes); got != expected { - t.Errorf("getReviewRounds(%v, %v) = %d, expected %d", commits, notes, got, expected) - } -} diff --git a/backend/plugins/gitlab/tasks/mr_label_convertor.go b/backend/plugins/gitlab/tasks/mr_label_convertor.go index 934085e8163..5ccc5009f8c 100644 --- a/backend/plugins/gitlab/tasks/mr_label_convertor.go +++ b/backend/plugins/gitlab/tasks/mr_label_convertor.go @@ -18,14 +18,12 @@ limitations under the License. package tasks import ( - "reflect" - "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer/code" "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" "github.com/apache/incubator-devlake/core/plugin" - helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/gitlab/models" ) @@ -42,35 +40,30 @@ var ConvertMrLabelsMeta = plugin.SubTaskMeta{ Dependencies: []*plugin.SubTaskMeta{&ConvertIssueLabelsMeta}, } -func ConvertMrLabels(taskCtx plugin.SubTaskContext) errors.Error { - db := taskCtx.GetDal() - rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_MERGE_REQUEST_TABLE) - projectId := data.Options.ProjectId - clauses := []dal.Clause{ - dal.Select("*"), - dal.From(&models.GitlabMrLabel{}), - dal.Join(`left join _tool_gitlab_merge_requests on - _tool_gitlab_merge_requests.gitlab_id = _tool_gitlab_mr_labels.mr_id`), - dal.Where(`_tool_gitlab_merge_requests.project_id = ? - and _tool_gitlab_merge_requests.connection_id = ?`, - projectId, data.Options.ConnectionId), - dal.Orderby("mr_id ASC"), - } - - cursor, err := db.Cursor(clauses...) - if err != nil { - return err - } - defer cursor.Close() +func ConvertMrLabels(subtaskCtx plugin.SubTaskContext) errors.Error { + subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_MERGE_REQUEST_TABLE) + db := subtaskCtx.GetDal() mrIdGen := didgen.NewDomainIdGenerator(&models.GitlabMergeRequest{}) - converter, err := helper.NewDataConverter(helper.DataConverterArgs{ - RawDataSubTaskArgs: *rawDataSubTaskArgs, - InputRowType: reflect.TypeOf(models.GitlabMrLabel{}), - Input: cursor, - Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { - mrLabel := inputRow.(*models.GitlabMrLabel) + converter, err := api.NewStatefulDataConverter[models.GitlabMrLabel](&api.StatefulDataConverterArgs[models.GitlabMrLabel]{ + SubtaskCommonArgs: subtaskCommonArgs, + Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { + clauses := []dal.Clause{ + dal.Select("c.*"), + dal.From("_tool_gitlab_mr_labels c"), + dal.Join(`LEFT JOIN _tool_gitlab_merge_requests mr ON mr.gitlab_id = c.merge_request_id`), + dal.Where(`mr.project_id = ? and mr.connection_id = ?`, data.Options.ProjectId, data.Options.ConnectionId), + } + if stateManager.IsIncremental() { + since := stateManager.GetSince() + if since != nil { + clauses = append(clauses, dal.Where("c.updated_at >= ? ", since)) + } + } + return db.Cursor(clauses...) + }, + Convert: func(mrLabel *models.GitlabMrLabel) ([]interface{}, errors.Error) { domainIssueLabel := &code.PullRequestLabel{ PullRequestId: mrIdGen.Generate(data.Options.ConnectionId, mrLabel.MrId), LabelName: mrLabel.LabelName, diff --git a/backend/plugins/gitlab/tasks/mr_note_extractor.go b/backend/plugins/gitlab/tasks/mr_note_extractor.go index a27d1141ce7..218b741fc4e 100644 --- a/backend/plugins/gitlab/tasks/mr_note_extractor.go +++ b/backend/plugins/gitlab/tasks/mr_note_extractor.go @@ -57,11 +57,11 @@ var ExtractApiMrNotesMeta = plugin.SubTaskMeta{ Dependencies: []*plugin.SubTaskMeta{&CollectApiMrNotesMeta}, } -func ExtractApiMergeRequestsNotes(taskCtx plugin.SubTaskContext) errors.Error { - rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_MERGE_REQUEST_NOTES_TABLE) +func ExtractApiMergeRequestsNotes(subtaskCtx plugin.SubTaskContext) errors.Error { + subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_MERGE_REQUEST_NOTES_TABLE) - extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ - RawDataSubTaskArgs: *rawDataSubTaskArgs, + extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs{ + SubtaskCommonArgs: subtaskCommonArgs, Extract: func(row *api.RawData) ([]interface{}, errors.Error) { mrNote := &MergeRequestNote{} err := errors.Convert(json.Unmarshal(row.Data, mrNote)) diff --git a/backend/plugins/gitlab/tasks/mr_reviewer_convertor.go b/backend/plugins/gitlab/tasks/mr_reviewer_convertor.go index b702de5733b..776c06c0b6f 100644 --- a/backend/plugins/gitlab/tasks/mr_reviewer_convertor.go +++ b/backend/plugins/gitlab/tasks/mr_reviewer_convertor.go @@ -18,14 +18,12 @@ limitations under the License. package tasks import ( - "reflect" - "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer/code" "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" "github.com/apache/incubator-devlake/core/plugin" - helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/gitlab/models" ) @@ -42,36 +40,31 @@ var ConvertMrReviewersMeta = plugin.SubTaskMeta{ Dependencies: []*plugin.SubTaskMeta{&ExtractApiMergeRequestDetailsMeta}, } -func ConvertMrReviewers(taskCtx plugin.SubTaskContext) errors.Error { - db := taskCtx.GetDal() - rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_MERGE_REQUEST_TABLE) - projectId := data.Options.ProjectId - clauses := []dal.Clause{ - dal.Select("_tool_gitlab_reviewers.*"), - dal.From(&models.GitlabReviewer{}), - dal.Join(`left join _tool_gitlab_merge_requests mr on - mr.gitlab_id = _tool_gitlab_reviewers.merge_request_id`), - dal.Where(`mr.project_id = ? - and mr.connection_id = ?`, - projectId, data.Options.ConnectionId), - dal.Orderby("_tool_gitlab_reviewers.merge_request_id ASC"), - } - - cursor, err := db.Cursor(clauses...) - if err != nil { - return err - } - defer cursor.Close() +func ConvertMrReviewers(subtaskCtx plugin.SubTaskContext) errors.Error { + subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_MERGE_REQUEST_TABLE) + db := subtaskCtx.GetDal() mrIdGen := didgen.NewDomainIdGenerator(&models.GitlabMergeRequest{}) accountIdGen := didgen.NewDomainIdGenerator(&models.GitlabAccount{}) - converter, err := helper.NewDataConverter(helper.DataConverterArgs{ - RawDataSubTaskArgs: *rawDataSubTaskArgs, - InputRowType: reflect.TypeOf(models.GitlabReviewer{}), - Input: cursor, - Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { - mrReviewer := inputRow.(*models.GitlabReviewer) + converter, err := api.NewStatefulDataConverter[models.GitlabReviewer](&api.StatefulDataConverterArgs[models.GitlabReviewer]{ + SubtaskCommonArgs: subtaskCommonArgs, + Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { + clauses := []dal.Clause{ + dal.Select("c.*"), + dal.From("_tool_gitlab_reviewers c"), + dal.Join(`LEFT JOIN _tool_gitlab_merge_requests mr ON mr.gitlab_id = c.merge_request_id`), + dal.Where(`mr.project_id = ? and mr.connection_id = ?`, data.Options.ProjectId, data.Options.ConnectionId), + } + if stateManager.IsIncremental() { + since := stateManager.GetSince() + if since != nil { + clauses = append(clauses, dal.Where("c.updated_at >= ? ", since)) + } + } + return db.Cursor(clauses...) + }, + Convert: func(mrReviewer *models.GitlabReviewer) ([]interface{}, errors.Error) { domainPrReviewer := &code.PullRequestReviewer{ PullRequestId: mrIdGen.Generate(data.Options.ConnectionId, mrReviewer.MergeRequestId), ReviewerId: accountIdGen.Generate(data.Options.ConnectionId, mrReviewer.ReviewerId), diff --git a/backend/plugins/gitlab/tasks/pipeline_commit_convertor.go b/backend/plugins/gitlab/tasks/pipeline_commit_convertor.go index 4984270d9de..1c73579e608 100644 --- a/backend/plugins/gitlab/tasks/pipeline_commit_convertor.go +++ b/backend/plugins/gitlab/tasks/pipeline_commit_convertor.go @@ -18,14 +18,12 @@ limitations under the License. package tasks import ( - "reflect" - "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer/devops" "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" "github.com/apache/incubator-devlake/core/plugin" - helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/gitlab/models" ) @@ -42,9 +40,9 @@ var ConvertPipelineCommitMeta = plugin.SubTaskMeta{ Dependencies: []*plugin.SubTaskMeta{&ConvertDetailPipelineMeta}, } -func ConvertPipelineCommits(taskCtx plugin.SubTaskContext) errors.Error { - db := taskCtx.GetDal() - data := taskCtx.GetData().(*GitlabTaskData) +func ConvertPipelineCommits(subtaskCtx plugin.SubTaskContext) errors.Error { + subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_PIPELINE_TABLE) + db := subtaskCtx.GetDal() repo := &models.GitlabProject{} err := db.First(repo, dal.Where("gitlab_id = ? and connection_id = ?", data.Options.ProjectId, data.Options.ConnectionId)) @@ -52,29 +50,24 @@ func ConvertPipelineCommits(taskCtx plugin.SubTaskContext) errors.Error { return err } - cursor, err := db.Cursor(dal.From(models.GitlabPipelineProject{}), - dal.Where("project_id = ? and connection_id = ?", data.Options.ProjectId, data.Options.ConnectionId)) - if err != nil { - return err - } - defer cursor.Close() - pipelineIdGen := didgen.NewDomainIdGenerator(&models.GitlabPipeline{}) - converter, err := helper.NewDataConverter(helper.DataConverterArgs{ - InputRowType: reflect.TypeOf(models.GitlabPipelineProject{}), - Input: cursor, - RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ - Ctx: taskCtx, - Params: models.GitlabApiParams{ - ConnectionId: data.Options.ConnectionId, - ProjectId: data.Options.ProjectId, - }, - Table: RAW_PIPELINE_TABLE, + converter, err := api.NewStatefulDataConverter[models.GitlabPipelineProject](&api.StatefulDataConverterArgs[models.GitlabPipelineProject]{ + SubtaskCommonArgs: subtaskCommonArgs, + Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { + clauses := []dal.Clause{ + dal.From(models.GitlabPipelineProject{}), + dal.Where("project_id = ? and connection_id = ?", data.Options.ProjectId, data.Options.ConnectionId), + } + if stateManager.IsIncremental() { + since := stateManager.GetSince() + if since != nil { + clauses = append(clauses, dal.Where("c.updated_at >= ? ", since)) + } + } + return db.Cursor(clauses...) }, - Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { - gitlabPipelineCommit := inputRow.(*models.GitlabPipelineProject) - + Convert: func(gitlabPipelineCommit *models.GitlabPipelineProject) ([]interface{}, errors.Error) { domainPipelineCommit := &devops.CiCDPipelineCommit{ PipelineId: pipelineIdGen.Generate(data.Options.ConnectionId, gitlabPipelineCommit.PipelineId), CommitSha: gitlabPipelineCommit.Sha, diff --git a/backend/plugins/gitlab/tasks/pipeline_detail_convertor.go b/backend/plugins/gitlab/tasks/pipeline_detail_convertor.go index 1b652274f76..87ca8f3f4a9 100644 --- a/backend/plugins/gitlab/tasks/pipeline_detail_convertor.go +++ b/backend/plugins/gitlab/tasks/pipeline_detail_convertor.go @@ -18,17 +18,16 @@ limitations under the License. package tasks import ( - "reflect" "time" "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/models/domainlayer" "github.com/apache/incubator-devlake/core/models/domainlayer/devops" "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" "github.com/apache/incubator-devlake/core/plugin" - helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/gitlab/models" ) @@ -45,34 +44,30 @@ var ConvertDetailPipelineMeta = plugin.SubTaskMeta{ Dependencies: []*plugin.SubTaskMeta{&ConvertCommitsMeta}, } -func ConvertDetailPipelines(taskCtx plugin.SubTaskContext) errors.Error { - db := taskCtx.GetDal() - data := taskCtx.GetData().(*GitlabTaskData) +func ConvertDetailPipelines(subtaskCtx plugin.SubTaskContext) errors.Error { + subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_PIPELINE_DETAILS_TABLE) - cursor, err := db.Cursor(dal.From(models.GitlabPipeline{}), - dal.Where("project_id = ? and connection_id = ?", data.Options.ProjectId, data.Options.ConnectionId)) - if err != nil { - return err - } - defer cursor.Close() + db := subtaskCtx.GetDal() pipelineIdGen := didgen.NewDomainIdGenerator(&models.GitlabPipeline{}) projectIdGen := didgen.NewDomainIdGenerator(&models.GitlabProject{}) - converter, err := helper.NewDataConverter(helper.DataConverterArgs{ - InputRowType: reflect.TypeOf(models.GitlabPipeline{}), - Input: cursor, - RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ - Ctx: taskCtx, - Params: models.GitlabApiParams{ - ConnectionId: data.Options.ConnectionId, - ProjectId: data.Options.ProjectId, - }, - Table: RAW_PIPELINE_DETAILS_TABLE, + converter, err := api.NewStatefulDataConverter[models.GitlabPipeline](&api.StatefulDataConverterArgs[models.GitlabPipeline]{ + SubtaskCommonArgs: subtaskCommonArgs, + Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { + clauses := []dal.Clause{ + dal.From(models.GitlabPipeline{}), + dal.Where("project_id = ? and connection_id = ?", data.Options.ProjectId, data.Options.ConnectionId), + } + if stateManager.IsIncremental() { + since := stateManager.GetSince() + if since != nil { + clauses = append(clauses, dal.Where("c.updated_at >= ? ", since)) + } + } + return db.Cursor(clauses...) }, - Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { - gitlabPipeline := inputRow.(*models.GitlabPipeline) - + Convert: func(gitlabPipeline *models.GitlabPipeline) ([]interface{}, errors.Error) { createdAt := time.Now() if gitlabPipeline.GitlabCreatedAt != nil { createdAt = *gitlabPipeline.GitlabCreatedAt diff --git a/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go b/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go index a5e597e2340..49146450ed8 100644 --- a/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go +++ b/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go @@ -41,11 +41,11 @@ var ExtractApiPipelineDetailsMeta = plugin.SubTaskMeta{ Dependencies: []*plugin.SubTaskMeta{&CollectApiPipelineDetailsMeta}, } -func ExtractApiPipelineDetails(taskCtx plugin.SubTaskContext) errors.Error { - rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PIPELINE_DETAILS_TABLE) +func ExtractApiPipelineDetails(subtaskCtx plugin.SubTaskContext) errors.Error { + subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_PIPELINE_DETAILS_TABLE) - extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ - RawDataSubTaskArgs: *rawDataSubTaskArgs, + extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs{ + SubtaskCommonArgs: subtaskCommonArgs, Extract: func(row *api.RawData) ([]interface{}, errors.Error) { // create gitlab commit gitlabApiPipeline := &ApiPipeline{} diff --git a/backend/plugins/gitlab/tasks/pipeline_extractor.go b/backend/plugins/gitlab/tasks/pipeline_extractor.go index 9cce27c805b..c7069a56a5b 100644 --- a/backend/plugins/gitlab/tasks/pipeline_extractor.go +++ b/backend/plugins/gitlab/tasks/pipeline_extractor.go @@ -69,11 +69,11 @@ var ExtractApiPipelinesMeta = plugin.SubTaskMeta{ Dependencies: []*plugin.SubTaskMeta{&CollectApiPipelinesMeta}, } -func ExtractApiPipelines(taskCtx plugin.SubTaskContext) errors.Error { - rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PIPELINE_TABLE) +func ExtractApiPipelines(subtaskCtx plugin.SubTaskContext) errors.Error { + subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_PIPELINE_TABLE) - extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ - RawDataSubTaskArgs: *rawDataSubTaskArgs, + extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs{ + SubtaskCommonArgs: subtaskCommonArgs, Extract: func(row *api.RawData) ([]interface{}, errors.Error) { // create gitlab commit gitlabApiPipeline := &ApiPipeline{} diff --git a/backend/plugins/gitlab/tasks/trigger_job_extractor.go b/backend/plugins/gitlab/tasks/trigger_job_extractor.go index f429f9b4d23..3ad3bb67aef 100644 --- a/backend/plugins/gitlab/tasks/trigger_job_extractor.go +++ b/backend/plugins/gitlab/tasks/trigger_job_extractor.go @@ -55,11 +55,11 @@ var ExtractApiTriggerJobsMeta = plugin.SubTaskMeta{ Dependencies: []*plugin.SubTaskMeta{&CollectApiTriggerJobsMeta}, } -func ExtractApiTriggerJobs(taskCtx plugin.SubTaskContext) errors.Error { - rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_TRIGGER_JOB_TABLE) +func ExtractApiTriggerJobs(subtaskCtx plugin.SubTaskContext) errors.Error { + subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_TRIGGER_JOB_TABLE) - extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ - RawDataSubTaskArgs: *rawDataSubTaskArgs, + extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs{ + SubtaskCommonArgs: subtaskCommonArgs, Extract: func(row *api.RawData) ([]interface{}, errors.Error) { // create gitlab commit gitlabApiTriggerJob := &ApiTriggerJob{} From 797a819c5075629989f726114818b61c21168940 Mon Sep 17 00:00:00 2001 From: Klesh Wong Date: Wed, 4 Sep 2024 16:27:37 +0800 Subject: [PATCH 04/23] fix: some gitlab subtasks are missing connection in left join clause --- backend/plugins/gitlab/tasks/issue_label_convertor.go | 11 ++++------- backend/plugins/gitlab/tasks/job_convertor.go | 6 +++--- backend/plugins/gitlab/tasks/job_extractor.go | 4 ++-- backend/plugins/gitlab/tasks/mr_commit_convertor.go | 2 +- backend/plugins/gitlab/tasks/mr_label_convertor.go | 2 +- backend/plugins/gitlab/tasks/mr_note_convertor.go | 2 +- backend/plugins/gitlab/tasks/mr_reviewer_convertor.go | 2 +- 7 files changed, 13 insertions(+), 16 deletions(-) diff --git a/backend/plugins/gitlab/tasks/issue_label_convertor.go b/backend/plugins/gitlab/tasks/issue_label_convertor.go index 4ffa1cc2f22..ddc67550ef5 100644 --- a/backend/plugins/gitlab/tasks/issue_label_convertor.go +++ b/backend/plugins/gitlab/tasks/issue_label_convertor.go @@ -58,13 +58,10 @@ func ConvertIssueLabels(subtaskCtx plugin.SubTaskContext) errors.Error { SubtaskCommonArgs: subtaskCommonArgs, Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { clauses := []dal.Clause{ - dal.Select("*"), - dal.From(&models.GitlabIssueLabel{}), - dal.Join(`left join _tool_gitlab_issues on - _tool_gitlab_issues.gitlab_id = _tool_gitlab_issue_labels.issue_id`), - dal.Where(`_tool_gitlab_issues.project_id = ? - and _tool_gitlab_issues.connection_id = ?`, - projectId, data.Options.ConnectionId), + dal.Select("l.*"), + dal.From("_tool_gitlab_issue_labels l"), + dal.Join(`LEFT JOIN _tool_gitlab_issues s ON s.gitlab_id = l.issue_id AND l.connection_id = s.connection_id`), + dal.Where(`s.project_id = ? AND s.connection_id = ?`, projectId, data.Options.ConnectionId), dal.Orderby("issue_id ASC"), } if stateManager.IsIncremental() { diff --git a/backend/plugins/gitlab/tasks/job_convertor.go b/backend/plugins/gitlab/tasks/job_convertor.go index 9e1042ac310..325de3f2831 100644 --- a/backend/plugins/gitlab/tasks/job_convertor.go +++ b/backend/plugins/gitlab/tasks/job_convertor.go @@ -44,9 +44,9 @@ var ConvertJobMeta = plugin.SubTaskMeta{ Dependencies: []*plugin.SubTaskMeta{&ConvertPipelineCommitMeta}, } -func ConvertJobs(taskCtx plugin.SubTaskContext) (err errors.Error) { - subtaskCommonArgs, data := CreateSubtaskCommonArgs(taskCtx, RAW_JOB_TABLE) - db := taskCtx.GetDal() +func ConvertJobs(subtaskCtx plugin.SubTaskContext) (err errors.Error) { + subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_JOB_TABLE) + db := subtaskCtx.GetDal() regexEnricher := data.RegexEnricher subtaskCommonArgs.SubtaskConfig = regexEnricher.PlainMap() diff --git a/backend/plugins/gitlab/tasks/job_extractor.go b/backend/plugins/gitlab/tasks/job_extractor.go index ec4cd5427c5..65bfb33aabf 100644 --- a/backend/plugins/gitlab/tasks/job_extractor.go +++ b/backend/plugins/gitlab/tasks/job_extractor.go @@ -59,8 +59,8 @@ var ExtractApiJobsMeta = plugin.SubTaskMeta{ Dependencies: []*plugin.SubTaskMeta{&CollectApiJobsMeta}, } -func ExtractApiJobs(taskCtx plugin.SubTaskContext) errors.Error { - subtaskCommonArgs, data := CreateSubtaskCommonArgs(taskCtx, RAW_JOB_TABLE) +func ExtractApiJobs(subtaskCtx plugin.SubTaskContext) errors.Error { + subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_JOB_TABLE) extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs{ SubtaskCommonArgs: subtaskCommonArgs, diff --git a/backend/plugins/gitlab/tasks/mr_commit_convertor.go b/backend/plugins/gitlab/tasks/mr_commit_convertor.go index 91ed65c4238..31c78a7fe41 100644 --- a/backend/plugins/gitlab/tasks/mr_commit_convertor.go +++ b/backend/plugins/gitlab/tasks/mr_commit_convertor.go @@ -52,7 +52,7 @@ func ConvertApiMergeRequestsCommits(subtaskCtx plugin.SubTaskContext) errors.Err clauses := []dal.Clause{ dal.Select("c.*"), dal.From("_tool_gitlab_mr_commits c"), - dal.Join(`LEFT JOIN _tool_gitlab_merge_requests mr ON mr.gitlab_id = c.merge_request_id`), + dal.Join(`LEFT JOIN _tool_gitlab_merge_requests mr ON mr.gitlab_id = c.merge_request_id AND c.connection_id = mr.connection_id`), dal.Where(`mr.project_id = ? and mr.connection_id = ?`, data.Options.ProjectId, data.Options.ConnectionId), } if stateManager.IsIncremental() { diff --git a/backend/plugins/gitlab/tasks/mr_label_convertor.go b/backend/plugins/gitlab/tasks/mr_label_convertor.go index 5ccc5009f8c..0682365be29 100644 --- a/backend/plugins/gitlab/tasks/mr_label_convertor.go +++ b/backend/plugins/gitlab/tasks/mr_label_convertor.go @@ -52,7 +52,7 @@ func ConvertMrLabels(subtaskCtx plugin.SubTaskContext) errors.Error { clauses := []dal.Clause{ dal.Select("c.*"), dal.From("_tool_gitlab_mr_labels c"), - dal.Join(`LEFT JOIN _tool_gitlab_merge_requests mr ON mr.gitlab_id = c.merge_request_id`), + dal.Join(`LEFT JOIN _tool_gitlab_merge_requests mr ON mr.gitlab_id = c.merge_request_id AND c.connection_id = mr.connection_id`), dal.Where(`mr.project_id = ? and mr.connection_id = ?`, data.Options.ProjectId, data.Options.ConnectionId), } if stateManager.IsIncremental() { diff --git a/backend/plugins/gitlab/tasks/mr_note_convertor.go b/backend/plugins/gitlab/tasks/mr_note_convertor.go index 27005e15eff..8c600851e17 100644 --- a/backend/plugins/gitlab/tasks/mr_note_convertor.go +++ b/backend/plugins/gitlab/tasks/mr_note_convertor.go @@ -55,7 +55,7 @@ func ConvertMergeRequestNote(subtaskCtx plugin.SubTaskContext) errors.Error { clauses := []dal.Clause{ dal.Select("c.*"), dal.From("_tool_gitlab_mr_comments c"), - dal.Join(`LEFT JOIN _tool_gitlab_merge_requests mr ON mr.gitlab_id = c.merge_request_id`), + dal.Join(`LEFT JOIN _tool_gitlab_merge_requests mr ON mr.gitlab_id = c.merge_request_id AND c.connection_id = mr.connection_id`), dal.Where(`mr.project_id = ? and mr.connection_id = ?`, data.Options.ProjectId, data.Options.ConnectionId), } if stateManager.IsIncremental() { diff --git a/backend/plugins/gitlab/tasks/mr_reviewer_convertor.go b/backend/plugins/gitlab/tasks/mr_reviewer_convertor.go index 776c06c0b6f..cf5728af858 100644 --- a/backend/plugins/gitlab/tasks/mr_reviewer_convertor.go +++ b/backend/plugins/gitlab/tasks/mr_reviewer_convertor.go @@ -53,7 +53,7 @@ func ConvertMrReviewers(subtaskCtx plugin.SubTaskContext) errors.Error { clauses := []dal.Clause{ dal.Select("c.*"), dal.From("_tool_gitlab_reviewers c"), - dal.Join(`LEFT JOIN _tool_gitlab_merge_requests mr ON mr.gitlab_id = c.merge_request_id`), + dal.Join(`LEFT JOIN _tool_gitlab_merge_requests mr ON mr.gitlab_id = c.merge_request_id AND c.connection_id = mr.connection_id`), dal.Where(`mr.project_id = ? and mr.connection_id = ?`, data.Options.ProjectId, data.Options.ConnectionId), } if stateManager.IsIncremental() { From 0bb3453d056de3a4e06110947a4595a22e3a0ae9 Mon Sep 17 00:00:00 2001 From: Klesh Wong Date: Wed, 4 Sep 2024 17:07:24 +0800 Subject: [PATCH 05/23] fix: test cases failed due to gitlab transformers support incr-mode --- backend/plugins/gitlab/e2e/mr_enrich_test.go | 79 ------------------- ..._gitlab_merge_requests_for_enrich_test.csv | 25 ------ ...tlab_merge_requests_for_mr_commit_test.csv | 12 +-- ...itlab_merge_requests_for_mr_notes_test.csv | 34 ++++---- .../plugins/gitlab/tasks/account_collector.go | 2 +- .../plugins/gitlab/tasks/issue_convertor.go | 2 +- .../gitlab/tasks/issue_label_convertor.go | 17 ++-- .../gitlab/tasks/mr_label_convertor.go | 2 +- 8 files changed, 31 insertions(+), 142 deletions(-) delete mode 100644 backend/plugins/gitlab/e2e/mr_enrich_test.go delete mode 100644 backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_merge_requests_for_enrich_test.csv diff --git a/backend/plugins/gitlab/e2e/mr_enrich_test.go b/backend/plugins/gitlab/e2e/mr_enrich_test.go deleted file mode 100644 index fd18003ccdd..00000000000 --- a/backend/plugins/gitlab/e2e/mr_enrich_test.go +++ /dev/null @@ -1,79 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2e - -import ( - "testing" - - "github.com/apache/incubator-devlake/helpers/e2ehelper" - "github.com/apache/incubator-devlake/plugins/gitlab/impl" - "github.com/apache/incubator-devlake/plugins/gitlab/models" - "github.com/apache/incubator-devlake/plugins/gitlab/tasks" -) - -func TestGitlabMrEnrichDataFlow(t *testing.T) { - - var gitlab impl.Gitlab - dataflowTester := e2ehelper.NewDataFlowTester(t, "gitlab", gitlab) - - taskData := &tasks.GitlabTaskData{ - Options: &tasks.GitlabOptions{ - ConnectionId: 1, - ProjectId: 12345678, - ScopeConfig: new(models.GitlabScopeConfig), - }, - } - - // import data table - dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_gitlab_merge_requests.csv", &models.GitlabMergeRequest{}) - dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_gitlab_mr_commits.csv", &models.GitlabMrCommit{}) - dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_gitlab_mr_notes.csv", &models.GitlabMrNote{}) - - // verify extraction - dataflowTester.Subtask(tasks.EnrichMergeRequestsMeta, taskData) - dataflowTester.VerifyTable( - models.GitlabMergeRequest{}, - "./snapshot_tables/_tool_gitlab_merge_requests_for_enrich_test.csv", - e2ehelper.ColumnWithRawData( - "connection_id", - "gitlab_id", - "iid", - "project_id", - "source_project_id", - "target_project_id", - "state", - "title", - "web_url", - "user_notes_count", - "work_in_progress", - "source_branch", - "target_branch", - "merge_commit_sha", - "merged_at", - "gitlab_created_at", - "closed_at", - "merged_by_username", - "description", - "author_username", - "author_user_id", - "component", - "first_comment_time", - "review_rounds", - ), - ) -} diff --git a/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_merge_requests_for_enrich_test.csv b/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_merge_requests_for_enrich_test.csv deleted file mode 100644 index 56804c7303b..00000000000 --- a/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_merge_requests_for_enrich_test.csv +++ /dev/null @@ -1,25 +0,0 @@ -connection_id,gitlab_id,iid,project_id,source_project_id,target_project_id,state,title,web_url,user_notes_count,work_in_progress,source_branch,target_branch,merge_commit_sha,merged_at,gitlab_created_at,closed_at,merged_by_username,description,author_username,author_user_id,component,first_comment_time,review_rounds,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark -1,32348491,1,12345678,12345678,12345678,merged,"Resolve ""Add documentation to snowflake spend package""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/1,1,0,1-add-documentation-to-snowflake-spend-package,master,da1d6dea48f5972ffc683da6cff30934e7d6c52c,2019-06-28T14:32:06.192+00:00,2019-06-28T05:21:43.743+00:00,,tayloramurphy,Closes #1,emilie,2295562,,,2,,,0, -1,35064956,3,12345678,13835497,12345678,merged,Update README to include steps to resolve a potential dbt-utils conflict,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/3,3,0,5-update-readme-to-include-steps-to-resolve-a-potential-dbt-utils-conflict,master,d678bea9d47b42eb13512d1c9d6a592d80b432d4,2019-08-26T14:15:27.922+00:00,2019-08-15T19:34:32.706+00:00,,emilie,Closes #5,martinguindon,3871284,,,1,,,0, -1,35841926,4,12345678,12345678,12345678,merged,"Resolve ""Config is not generic enough""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/4,0,0,4-config-is-not-generic-enough,master,e95b5db25e15a38e21d11cb45cc21bf17d5c407c,2019-08-26T15:37:50.105+00:00,2019-08-26T15:32:49.557+00:00,,emilie,Closes #4,emilie,2295562,,,1,,,0, -1,53445063,5,12345678,15706315,12345678,merged,Issue 3 Base model,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/5,20,0,issue_3,master,f2ee4cf121a328ce39723506dc18e4661941971a,2020-03-25T18:36:45.801+00:00,2020-03-24T12:46:15.891+00:00,,tayloramurphy,,nehiljain,783199,,,1,,,0, -1,53627854,6,12345678,15706063,12345678,merged,Update schema.yml typo in docs,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/6,0,0,patch-1,master,12dcc23a45adce0b12f8687438ec3a28274c7c30,2020-03-25T19:04:19.844+00:00,2020-03-25T19:02:16.747+00:00,,tayloramurphy,,nehiljain,783199,,,1,,,0, -1,55146687,8,12345678,12345678,12345678,merged,"Resolve ""Document release process""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/8,3,0,6-document-release-process,master,7c8245a3a5eda7f502737940aaf7944d99c58f2e,2020-04-08T20:52:11.150+00:00,2020-04-08T20:07:10.223+00:00,,emilie,Closes #6,m_walker,5212782,,,1,,,0, -1,55146787,9,12345678,15706315,12345678,opened,Issue 3: Transformed model for query performance,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/9,14,0,issue_3,master,,,2020-04-08T20:09:08.130+00:00,,,,nehiljain,783199,,,1,,,0, -1,58311001,10,12345678,12345678,12345678,merged,Update version in readme,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/10,0,0,emilie-master-patch-23079,master,66c0f1de49a0c876b8f93e8e0dce3327e766f59d,2020-05-11T17:09:20.603+00:00,2020-05-11T17:09:12.265+00:00,,emilie,,emilie,2295562,,,1,,,0, -1,62519057,11,12345678,19569570,12345678,opened,Clustering metering models,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/11,0,0,clustering-metering,master,,,2020-06-24T12:34:04.792+00:00,,,,jainnehil,842680,,,1,,,0, -1,65505080,12,12345678,12345678,12345678,merged,"Resolve ""Upgrade package for dbt 0.17""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/12,0,0,11-upgrade-package-for-dbt-0-17,master,9bfc136eb90802c2ce59956c34dde01bb3de0d50,2020-07-24T21:13:35.321+00:00,2020-07-24T17:47:08.238+00:00,,tayloramurphy,"Closes #11 - -* Upgrades to 0.17.0 format -* Formatting changes to be in line with GitLab SQL Style Guide",tayloramurphy,1942272,,,1,,,0, -1,68978485,13,12345678,15706315,12345678,closed,Include more snowflake qrt columns,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/13,0,0,include_more_snowflake_qrt_columns,master,,,2020-08-27T20:17:01.825+00:00,2020-08-27T20:20:08.150+00:00,,,aianus,2478227,,,1,,,0, -1,89243644,14,12345678,24539973,12345678,merged,Update README.md to use the newest version as an example,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/14,0,0,ThomasLaPiana-master-patch-93997,master,21840a7eadb58babe8aeae2960da851a3ed00ddc,2021-02-19T20:13:05.969+00:00,2021-02-19T20:12:14.302+00:00,,tayloramurphy,Update README.md to use the newest version as an example. The old version doesn't work with the current version of dbt,ThomasLaPiana,2061802,,,1,,,0, -1,110817220,16,12345678,28584714,12345678,merged,Update packages.yml to point to dbt-labs instead of fishtown,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/16,5,0,GJMcClintock-master-patch-24867,master,6f45b467c478df1c67d19cf6d4cbb8e05a710662,2021-08-12T06:12:54.329+00:00,2021-08-03T15:02:54.955+00:00,,vedprakash2021,With the company name change the old repo is deprecated.,GJMcClintock,9439881,,,1,,,0, -1,111383524,17,12345678,0,12345678,closed,The package name changed -> https://hub.getdbt.com/dbt-labs/dbt_utils/latest/,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/17,1,0,swiffer-master-patch-77533,master,,,2021-08-07T06:50:25.458+00:00,2021-08-07T06:51:14.933+00:00,,,swiffer,156402,,,1,,,0, -1,114994501,18,12345678,29298577,12345678,opened,Add support for Snowpipe usage monitoring,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/18,0,0,master,master,,,2021-09-01T21:15:30.334+00:00,,,Add models and docs for Snowflake pipes (Snowpipe) usage monitoring based on the views in Snowflake Usage schema,gary-beautypie,9635687,,,1,,,0, -1,135775405,19,12345678,32935405,12345678,opened,Updates for dbt 1.0,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/19,1,0,updates_for_dbt_1.0,master,,,2022-01-18T19:59:30.723+00:00,,,"This MR sets up the repo for dbt 1.0 -A few configs were renamed. - -Could a new release be made for dbt 1.0?",johnj4,10663622,,,1,,,0, -1,145012495,20,12345678,34491818,12345678,closed,Draft: Update dbt_project.yml,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/20,0,1,PedramNavid-master-patch-20645,master,,,2022-03-15T03:07:06.077+00:00,2022-03-15T03:07:22.665+00:00,,,PedramNavid,9722492,,,1,,,0, -1,158698019,21,12345678,12345678,12345678,opened,Draft: Corrections for dbt 1,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/21,0,1,updates_for_dbt_1_1,master,,,2022-06-03T09:24:53.707+00:00,,,Closes https://gitlab.com/gitlab-data/analytics/-/issues/12941,paul_armstrong,5618371,,,1,,,0, diff --git a/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_merge_requests_for_mr_commit_test.csv b/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_merge_requests_for_mr_commit_test.csv index 2ba28fadbaf..dc468a58ba5 100644 --- a/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_merge_requests_for_mr_commit_test.csv +++ b/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_merge_requests_for_mr_commit_test.csv @@ -1,5 +1,5 @@ -connection_id,gitlab_id,iid,project_id,source_project_id,target_project_id,state,title,web_url,user_notes_count,work_in_progress,source_branch,target_branch,merge_commit_sha,merged_at,gitlab_created_at,closed_at,merged_by_username,description,author_username,author_user_id,component,first_comment_time,review_rounds,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark -1,32348491,1,12345678,12345678,12345678,merged,"Resolve ""Add documentation to snowflake spend package""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/1,1,0,1-add-documentation-to-snowflake-spend-package,master,da1d6dea48f5972ffc683da6cff30934e7d6c52c,2019-06-28T14:32:06.192+00:00,2019-06-28T05:21:43.743+00:00,,tayloramurphy,Closes #1,emilie,2295562,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,1, +connection_id,gitlab_id,iid,project_id,source_project_id,target_project_id,state,title,web_url,user_notes_count,work_in_progress,source_branch,target_branch,merge_commit_sha,merged_at,gitlab_created_at,closed_at,merged_by_username,description,author_username,author_user_id,component,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark +1,32348491,1,12345678,12345678,12345678,merged,"Resolve ""Add documentation to snowflake spend package""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/1,1,0,1-add-documentation-to-snowflake-spend-package,master,da1d6dea48f5972ffc683da6cff30934e7d6c52c,2019-06-28T14:32:06.192+00:00,2019-06-28T05:21:43.743+00:00,,tayloramurphy,Closes #1,emilie,2295562,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,1, 1,34300715,2,12345678,12345678,12345678,merged,"Resolve ""Handle multiple prices and date ranges""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/2,9,0,2-handle-multiple-prices-and-date-ranges,master,2fe82576fdbd0a43a2d5e02c17c6dc6ce3e188ee,2019-08-04T22:24:41.219+00:00,2019-08-01T21:36:45.020+00:00,,emilie,"Closes #2 * Adds a model to amortize the snowflake_contract_rates.csv properly @@ -8,7 +8,7 @@ connection_id,gitlab_id,iid,project_id,source_project_id,target_project_id,state Testing by running dbt seed on https://gitlab.com/gitlab-data/analytics/merge_requests/1351/diffs and pinning dbt deps to this branch `2-handle...`. -All models ran fine and the data looks great in my scratch schema!",tayloramurphy,1942272,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,2, -1,35064956,3,12345678,13835497,12345678,merged,Update README to include steps to resolve a potential dbt-utils conflict,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/3,3,0,5-update-readme-to-include-steps-to-resolve-a-potential-dbt-utils-conflict,master,d678bea9d47b42eb13512d1c9d6a592d80b432d4,2019-08-26T14:15:27.922+00:00,2019-08-15T19:34:32.706+00:00,,emilie,Closes #5,martinguindon,3871284,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,3, -1,35841926,4,12345678,12345678,12345678,merged,"Resolve ""Config is not generic enough""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/4,0,0,4-config-is-not-generic-enough,master,e95b5db25e15a38e21d11cb45cc21bf17d5c407c,2019-08-26T15:37:50.105+00:00,2019-08-26T15:32:49.557+00:00,,emilie,Closes #4,emilie,2295562,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,4, -1,53445063,5,12345678,15706315,12345678,merged,Issue 3 Base model,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/5,20,0,issue_3,master,f2ee4cf121a328ce39723506dc18e4661941971a,2020-03-25T18:36:45.801+00:00,2020-03-24T12:46:15.891+00:00,,tayloramurphy,,nehiljain,783199,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,5, +All models ran fine and the data looks great in my scratch schema!",tayloramurphy,1942272,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,2, +1,35064956,3,12345678,13835497,12345678,merged,Update README to include steps to resolve a potential dbt-utils conflict,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/3,3,0,5-update-readme-to-include-steps-to-resolve-a-potential-dbt-utils-conflict,master,d678bea9d47b42eb13512d1c9d6a592d80b432d4,2019-08-26T14:15:27.922+00:00,2019-08-15T19:34:32.706+00:00,,emilie,Closes #5,martinguindon,3871284,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,3, +1,35841926,4,12345678,12345678,12345678,merged,"Resolve ""Config is not generic enough""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/4,0,0,4-config-is-not-generic-enough,master,e95b5db25e15a38e21d11cb45cc21bf17d5c407c,2019-08-26T15:37:50.105+00:00,2019-08-26T15:32:49.557+00:00,,emilie,Closes #4,emilie,2295562,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,4, +1,53445063,5,12345678,15706315,12345678,merged,Issue 3 Base model,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/5,20,0,issue_3,master,f2ee4cf121a328ce39723506dc18e4661941971a,2020-03-25T18:36:45.801+00:00,2020-03-24T12:46:15.891+00:00,,tayloramurphy,,nehiljain,783199,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,5, diff --git a/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_merge_requests_for_mr_notes_test.csv b/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_merge_requests_for_mr_notes_test.csv index fe38d4100c2..05d7a0c7df5 100644 --- a/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_merge_requests_for_mr_notes_test.csv +++ b/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_merge_requests_for_mr_notes_test.csv @@ -1,6 +1,6 @@ -connection_id,gitlab_id,iid,project_id,source_project_id,target_project_id,state,title,web_url,user_notes_count,work_in_progress,source_branch,target_branch,merge_commit_sha,merged_at,gitlab_created_at,closed_at,merged_by_username,description,author_username,author_user_id,component,first_comment_time,review_rounds,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark -1,15869219,37,12345678,12345678,12345678,opened,Draft: Corrections for dbt 1,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/21,0,1,updates_for_dbt_1_1,master,,,2022-06-03T09:24:53.707+00:00,,,Closes https://gitlab.com/gitlab-data/analytics/-/issues/12941,paul_armstrong,5618371,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,21, -1,32348491,1,12345678,12345678,12345678,merged,"Resolve ""Add documentation to snowflake spend package""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/1,1,0,1-add-documentation-to-snowflake-spend-package,master,da1d6dea48f5972ffc683da6cff30934e7d6c52c,2019-06-28T14:32:06.192+00:00,2019-06-28T05:21:43.743+00:00,,tayloramurphy,Closes #1,emilie,2295562,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,1, +connection_id,gitlab_id,iid,project_id,source_project_id,target_project_id,state,title,web_url,user_notes_count,work_in_progress,source_branch,target_branch,merge_commit_sha,merged_at,gitlab_created_at,closed_at,merged_by_username,description,author_username,author_user_id,component,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark +1,15869219,37,12345678,12345678,12345678,opened,Draft: Corrections for dbt 1,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/21,0,1,updates_for_dbt_1_1,master,,,2022-06-03T09:24:53.707+00:00,,,Closes https://gitlab.com/gitlab-data/analytics/-/issues/12941,paul_armstrong,5618371,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,21, +1,32348491,1,12345678,12345678,12345678,merged,"Resolve ""Add documentation to snowflake spend package""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/1,1,0,1-add-documentation-to-snowflake-spend-package,master,da1d6dea48f5972ffc683da6cff30934e7d6c52c,2019-06-28T14:32:06.192+00:00,2019-06-28T05:21:43.743+00:00,,tayloramurphy,Closes #1,emilie,2295562,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,1, 1,34300715,2,12345678,12345678,12345678,merged,"Resolve ""Handle multiple prices and date ranges""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/2,9,0,2-handle-multiple-prices-and-date-ranges,master,2fe82576fdbd0a43a2d5e02c17c6dc6ce3e188ee,2019-08-04T22:24:41.219+00:00,2019-08-01T21:36:45.020+00:00,,emilie,"Closes #2 * Adds a model to amortize the snowflake_contract_rates.csv properly @@ -9,23 +9,23 @@ connection_id,gitlab_id,iid,project_id,source_project_id,target_project_id,state Testing by running dbt seed on https://gitlab.com/gitlab-data/analytics/merge_requests/1351/diffs and pinning dbt deps to this branch `2-handle...`. -All models ran fine and the data looks great in my scratch schema!",tayloramurphy,1942272,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,2, -1,35064956,3,12345678,13835497,12345678,merged,Update README to include steps to resolve a potential dbt-utils conflict,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/3,3,0,5-update-readme-to-include-steps-to-resolve-a-potential-dbt-utils-conflict,master,d678bea9d47b42eb13512d1c9d6a592d80b432d4,2019-08-26T14:15:27.922+00:00,2019-08-15T19:34:32.706+00:00,,emilie,Closes #5,martinguindon,3871284,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,3, -1,35841926,4,12345678,12345678,12345678,merged,"Resolve ""Config is not generic enough""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/4,0,0,4-config-is-not-generic-enough,master,e95b5db25e15a38e21d11cb45cc21bf17d5c407c,2019-08-26T15:37:50.105+00:00,2019-08-26T15:32:49.557+00:00,,emilie,Closes #4,emilie,2295562,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,4, -1,53445063,5,12345678,15706315,12345678,merged,Issue 3 Base model,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/5,20,0,issue_3,master,f2ee4cf121a328ce39723506dc18e4661941971a,2020-03-25T18:36:45.801+00:00,2020-03-24T12:46:15.891+00:00,,tayloramurphy,,nehiljain,783199,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,5, -1,53627854,6,12345678,15706063,12345678,merged,Update schema.yml typo in docs,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/6,0,0,patch-1,master,12dcc23a45adce0b12f8687438ec3a28274c7c30,2020-03-25T19:04:19.844+00:00,2020-03-25T19:02:16.747+00:00,,tayloramurphy,,nehiljain,783199,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,6, -1,53640548,7,12345678,12345678,12345678,merged,"Resolve ""Add sources and cleanup structure""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/7,5,0,7-add-sources-and-cleanup-structure,master,1161de80448bc66072c3ea39600d9940dde87a5b,2020-04-07T12:51:50.096+00:00,2020-03-25T21:58:40.647+00:00,,emilie,Closes #7,tayloramurphy,1942272,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,7, -1,55146687,8,12345678,12345678,12345678,merged,"Resolve ""Document release process""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/8,3,0,6-document-release-process,master,7c8245a3a5eda7f502737940aaf7944d99c58f2e,2020-04-08T20:52:11.150+00:00,2020-04-08T20:07:10.223+00:00,,emilie,Closes #6,m_walker,5212782,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,8, -1,55146787,9,12345678,15706315,12345678,opened,Issue 3: Transformed model for query performance,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/9,14,0,issue_3,master,,,2020-04-08T20:09:08.130+00:00,,,,nehiljain,783199,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,9, -1,58311001,10,12345678,12345678,12345678,merged,Update version in readme,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/10,0,0,emilie-master-patch-23079,master,66c0f1de49a0c876b8f93e8e0dce3327e766f59d,2020-05-11T17:09:20.603+00:00,2020-05-11T17:09:12.265+00:00,,emilie,,emilie,2295562,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,10, -1,62519057,11,12345678,19569570,12345678,opened,Clustering metering models,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/11,0,0,clustering-metering,master,,,2020-06-24T12:34:04.792+00:00,,,,jainnehil,842680,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,11, +All models ran fine and the data looks great in my scratch schema!",tayloramurphy,1942272,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,2, +1,35064956,3,12345678,13835497,12345678,merged,Update README to include steps to resolve a potential dbt-utils conflict,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/3,3,0,5-update-readme-to-include-steps-to-resolve-a-potential-dbt-utils-conflict,master,d678bea9d47b42eb13512d1c9d6a592d80b432d4,2019-08-26T14:15:27.922+00:00,2019-08-15T19:34:32.706+00:00,,emilie,Closes #5,martinguindon,3871284,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,3, +1,35841926,4,12345678,12345678,12345678,merged,"Resolve ""Config is not generic enough""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/4,0,0,4-config-is-not-generic-enough,master,e95b5db25e15a38e21d11cb45cc21bf17d5c407c,2019-08-26T15:37:50.105+00:00,2019-08-26T15:32:49.557+00:00,,emilie,Closes #4,emilie,2295562,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,4, +1,53445063,5,12345678,15706315,12345678,merged,Issue 3 Base model,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/5,20,0,issue_3,master,f2ee4cf121a328ce39723506dc18e4661941971a,2020-03-25T18:36:45.801+00:00,2020-03-24T12:46:15.891+00:00,,tayloramurphy,,nehiljain,783199,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,5, +1,53627854,6,12345678,15706063,12345678,merged,Update schema.yml typo in docs,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/6,0,0,patch-1,master,12dcc23a45adce0b12f8687438ec3a28274c7c30,2020-03-25T19:04:19.844+00:00,2020-03-25T19:02:16.747+00:00,,tayloramurphy,,nehiljain,783199,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,6, +1,53640548,7,12345678,12345678,12345678,merged,"Resolve ""Add sources and cleanup structure""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/7,5,0,7-add-sources-and-cleanup-structure,master,1161de80448bc66072c3ea39600d9940dde87a5b,2020-04-07T12:51:50.096+00:00,2020-03-25T21:58:40.647+00:00,,emilie,Closes #7,tayloramurphy,1942272,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,7, +1,55146687,8,12345678,12345678,12345678,merged,"Resolve ""Document release process""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/8,3,0,6-document-release-process,master,7c8245a3a5eda7f502737940aaf7944d99c58f2e,2020-04-08T20:52:11.150+00:00,2020-04-08T20:07:10.223+00:00,,emilie,Closes #6,m_walker,5212782,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,8, +1,55146787,9,12345678,15706315,12345678,opened,Issue 3: Transformed model for query performance,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/9,14,0,issue_3,master,,,2020-04-08T20:09:08.130+00:00,,,,nehiljain,783199,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,9, +1,58311001,10,12345678,12345678,12345678,merged,Update version in readme,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/10,0,0,emilie-master-patch-23079,master,66c0f1de49a0c876b8f93e8e0dce3327e766f59d,2020-05-11T17:09:20.603+00:00,2020-05-11T17:09:12.265+00:00,,emilie,,emilie,2295562,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,10, +1,62519057,11,12345678,19569570,12345678,opened,Clustering metering models,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/11,0,0,clustering-metering,master,,,2020-06-24T12:34:04.792+00:00,,,,jainnehil,842680,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,11, 1,65505080,12,12345678,12345678,12345678,merged,"Resolve ""Upgrade package for dbt 0.17""",https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/12,0,0,11-upgrade-package-for-dbt-0-17,master,9bfc136eb90802c2ce59956c34dde01bb3de0d50,2020-07-24T21:13:35.321+00:00,2020-07-24T17:47:08.238+00:00,,tayloramurphy,"Closes #11 * Upgrades to 0.17.0 format -* Formatting changes to be in line with GitLab SQL Style Guide",tayloramurphy,1942272,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,12, +* Formatting changes to be in line with GitLab SQL Style Guide",tayloramurphy,1942272,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,12, 1,135772105,30,12345678,32935405,12345678,opened,Updates for dbt 1.0,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/19,1,0,updates_for_dbt_1.0,master,,,2022-01-18T19:59:30.723+00:00,,,"This MR sets up the repo for dbt 1.0 A few configs were renamed. -Could a new release be made for dbt 1.0?",johnj4,10663622,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,19, -1,145032495,46,12345678,34491818,12345678,closed,Draft: Update dbt_project.yml,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/20,0,1,PedramNavid-master-patch-20645,master,,,2022-03-15T03:07:06.077+00:00,2022-03-15T03:07:22.665+00:00,,,PedramNavid,9722492,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,20, -1,1149942101,29,12345678,29298577,12345678,opened,Add support for Snowpipe usage monitoring,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/18,0,0,master,master,,,2021-09-01T21:15:30.334+00:00,,,Add models and docs for Snowflake pipes (Snowpipe) usage monitoring based on the views in Snowflake Usage schema,gary-beautypie,9635687,,,0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,18, +Could a new release be made for dbt 1.0?",johnj4,10663622,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,19, +1,145032495,46,12345678,34491818,12345678,closed,Draft: Update dbt_project.yml,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/20,0,1,PedramNavid-master-patch-20645,master,,,2022-03-15T03:07:06.077+00:00,2022-03-15T03:07:22.665+00:00,,,PedramNavid,9722492,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,20, +1,1149942101,29,12345678,29298577,12345678,opened,Add support for Snowpipe usage monitoring,https://gitlab.com/gitlab-data/snowflake_spend/-/merge_requests/18,0,0,master,master,,,2021-09-01T21:15:30.334+00:00,,,Add models and docs for Snowflake pipes (Snowpipe) usage monitoring based on the views in Snowflake Usage schema,gary-beautypie,9635687,,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_requests,18, diff --git a/backend/plugins/gitlab/tasks/account_collector.go b/backend/plugins/gitlab/tasks/account_collector.go index 52ea1a72d84..da2e568c19c 100644 --- a/backend/plugins/gitlab/tasks/account_collector.go +++ b/backend/plugins/gitlab/tasks/account_collector.go @@ -43,7 +43,7 @@ var CollectAccountsMeta = plugin.SubTaskMeta{ EnabledByDefault: true, Description: "collect gitlab users, does not support either timeFilter or diffSync.", DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, - Dependencies: []*plugin.SubTaskMeta{&EnrichMergeRequestsMeta}, + Dependencies: []*plugin.SubTaskMeta{}, } func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error { diff --git a/backend/plugins/gitlab/tasks/issue_convertor.go b/backend/plugins/gitlab/tasks/issue_convertor.go index 5f120c52ec1..236f899de6e 100644 --- a/backend/plugins/gitlab/tasks/issue_convertor.go +++ b/backend/plugins/gitlab/tasks/issue_convertor.go @@ -58,7 +58,7 @@ func ConvertIssues(subtaskCtx plugin.SubTaskContext) errors.Error { Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { clauses := []dal.Clause{ dal.From(&models.GitlabIssue{}), - dal.Where("connection_id = ? AND gitlab_id = ?", data.Options.ConnectionId, data.Options.ProjectId), + dal.Where("connection_id = ? AND project_id = ?", data.Options.ConnectionId, data.Options.ProjectId), } if stateManager.IsIncremental() { since := stateManager.GetSince() diff --git a/backend/plugins/gitlab/tasks/issue_label_convertor.go b/backend/plugins/gitlab/tasks/issue_label_convertor.go index ddc67550ef5..3799335916f 100644 --- a/backend/plugins/gitlab/tasks/issue_label_convertor.go +++ b/backend/plugins/gitlab/tasks/issue_label_convertor.go @@ -42,16 +42,9 @@ var ConvertIssueLabelsMeta = plugin.SubTaskMeta{ func ConvertIssueLabels(subtaskCtx plugin.SubTaskContext) errors.Error { subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_ISSUE_TABLE) + db := subtaskCtx.GetDal() projectId := data.Options.ProjectId - clauses := []dal.Clause{} - - cursor, err := db.Cursor(clauses...) - if err != nil { - return err - } - defer cursor.Close() - issueIdGen := didgen.NewDomainIdGenerator(&models.GitlabIssue{}) converter, err := api.NewStatefulDataConverter[models.GitlabIssueLabel](&api.StatefulDataConverterArgs[models.GitlabIssueLabel]{ @@ -60,14 +53,14 @@ func ConvertIssueLabels(subtaskCtx plugin.SubTaskContext) errors.Error { clauses := []dal.Clause{ dal.Select("l.*"), dal.From("_tool_gitlab_issue_labels l"), - dal.Join(`LEFT JOIN _tool_gitlab_issues s ON s.gitlab_id = l.issue_id AND l.connection_id = s.connection_id`), - dal.Where(`s.project_id = ? AND s.connection_id = ?`, projectId, data.Options.ConnectionId), - dal.Orderby("issue_id ASC"), + dal.Join("LEFT JOIN _tool_gitlab_issues s ON s.gitlab_id = l.issue_id AND l.connection_id = s.connection_id"), + dal.Where("s.project_id = ? AND s.connection_id = ?", projectId, data.Options.ConnectionId), + // dal.Orderby("s.issue_id ASC"), } if stateManager.IsIncremental() { since := stateManager.GetSince() if since != nil { - clauses = append(clauses, dal.Where("_tool_gitlab_issues.updated_at >= ? ", since)) + clauses = append(clauses, dal.Where("l.updated_at >= ? ", since)) } } return db.Cursor(clauses...) diff --git a/backend/plugins/gitlab/tasks/mr_label_convertor.go b/backend/plugins/gitlab/tasks/mr_label_convertor.go index 0682365be29..ccb0a3bc0fd 100644 --- a/backend/plugins/gitlab/tasks/mr_label_convertor.go +++ b/backend/plugins/gitlab/tasks/mr_label_convertor.go @@ -52,7 +52,7 @@ func ConvertMrLabels(subtaskCtx plugin.SubTaskContext) errors.Error { clauses := []dal.Clause{ dal.Select("c.*"), dal.From("_tool_gitlab_mr_labels c"), - dal.Join(`LEFT JOIN _tool_gitlab_merge_requests mr ON mr.gitlab_id = c.merge_request_id AND c.connection_id = mr.connection_id`), + dal.Join(`LEFT JOIN _tool_gitlab_merge_requests mr ON mr.gitlab_id = c.mr_id AND c.connection_id = mr.connection_id`), dal.Where(`mr.project_id = ? and mr.connection_id = ?`, data.Options.ProjectId, data.Options.ConnectionId), } if stateManager.IsIncremental() { From ec4ca35224d809ea18e8fceed2a8b43fdb804bfa Mon Sep 17 00:00:00 2001 From: Klesh Wong Date: Wed, 4 Sep 2024 17:19:31 +0800 Subject: [PATCH 06/23] fix: linting --- backend/plugins/gitlab/tasks/job_convertor.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/backend/plugins/gitlab/tasks/job_convertor.go b/backend/plugins/gitlab/tasks/job_convertor.go index 325de3f2831..acbec7df45d 100644 --- a/backend/plugins/gitlab/tasks/job_convertor.go +++ b/backend/plugins/gitlab/tasks/job_convertor.go @@ -28,7 +28,6 @@ import ( "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/gitlab/models" - gitlabModels "github.com/apache/incubator-devlake/plugins/gitlab/models" ) func init() { @@ -50,14 +49,14 @@ func ConvertJobs(subtaskCtx plugin.SubTaskContext) (err errors.Error) { regexEnricher := data.RegexEnricher subtaskCommonArgs.SubtaskConfig = regexEnricher.PlainMap() - jobIdGen := didgen.NewDomainIdGenerator(&gitlabModels.GitlabJob{}) - projectIdGen := didgen.NewDomainIdGenerator(&gitlabModels.GitlabProject{}) - pipelineIdGen := didgen.NewDomainIdGenerator(&gitlabModels.GitlabPipeline{}) + jobIdGen := didgen.NewDomainIdGenerator(&models.GitlabJob{}) + projectIdGen := didgen.NewDomainIdGenerator(&models.GitlabProject{}) + pipelineIdGen := didgen.NewDomainIdGenerator(&models.GitlabPipeline{}) converter, err := api.NewStatefulDataConverter[models.GitlabJob](&api.StatefulDataConverterArgs[models.GitlabJob]{ SubtaskCommonArgs: subtaskCommonArgs, Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { clauses := []dal.Clause{ - dal.From(gitlabModels.GitlabJob{}), + dal.From(models.GitlabJob{}), dal.Where("project_id = ? and connection_id = ?", data.Options.ProjectId, data.Options.ConnectionId), } if stateManager.IsIncremental() { From 37611a5e5f4bbef4123e37364269f15ea3bd2713 Mon Sep 17 00:00:00 2001 From: Klesh Wong Date: Thu, 5 Sep 2024 17:13:49 +0800 Subject: [PATCH 07/23] fix: do not collect all accounts from jihulab.com --- backend/plugins/gitlab/tasks/account_collector.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/plugins/gitlab/tasks/account_collector.go b/backend/plugins/gitlab/tasks/account_collector.go index da2e568c19c..e0849858971 100644 --- a/backend/plugins/gitlab/tasks/account_collector.go +++ b/backend/plugins/gitlab/tasks/account_collector.go @@ -58,7 +58,7 @@ func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error { } // Collect all users if endpoint is private gitlab instance - if !strings.HasPrefix(data.ApiClient.GetEndpoint(), "https://gitlab.com") { + if !strings.HasPrefix(data.ApiClient.GetEndpoint(), "https://gitlab.com") && !strings.HasPrefix(data.ApiClient.GetEndpoint(), "https://jihulab.com") { urlTemplate = "/users" } From 25db78e63fd30e451605cd11189d4da89a689af5 Mon Sep 17 00:00:00 2001 From: Klesh Wong Date: Thu, 5 Sep 2024 17:16:39 +0800 Subject: [PATCH 08/23] fix: typo --- backend/plugins/gitlab/tasks/pipeline_commit_convertor.go | 2 +- backend/plugins/gitlab/tasks/pipeline_detail_convertor.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/plugins/gitlab/tasks/pipeline_commit_convertor.go b/backend/plugins/gitlab/tasks/pipeline_commit_convertor.go index 1c73579e608..b1cd4e0a45a 100644 --- a/backend/plugins/gitlab/tasks/pipeline_commit_convertor.go +++ b/backend/plugins/gitlab/tasks/pipeline_commit_convertor.go @@ -62,7 +62,7 @@ func ConvertPipelineCommits(subtaskCtx plugin.SubTaskContext) errors.Error { if stateManager.IsIncremental() { since := stateManager.GetSince() if since != nil { - clauses = append(clauses, dal.Where("c.updated_at >= ? ", since)) + clauses = append(clauses, dal.Where("updated_at >= ? ", since)) } } return db.Cursor(clauses...) diff --git a/backend/plugins/gitlab/tasks/pipeline_detail_convertor.go b/backend/plugins/gitlab/tasks/pipeline_detail_convertor.go index 87ca8f3f4a9..6cc8c5358c1 100644 --- a/backend/plugins/gitlab/tasks/pipeline_detail_convertor.go +++ b/backend/plugins/gitlab/tasks/pipeline_detail_convertor.go @@ -62,7 +62,7 @@ func ConvertDetailPipelines(subtaskCtx plugin.SubTaskContext) errors.Error { if stateManager.IsIncremental() { since := stateManager.GetSince() if since != nil { - clauses = append(clauses, dal.Where("c.updated_at >= ? ", since)) + clauses = append(clauses, dal.Where("updated_at >= ? ", since)) } } return db.Cursor(clauses...) From 5045741dbddd88a488f8c356b13bfb5843a9cc0f Mon Sep 17 00:00:00 2001 From: Klesh Wong Date: Thu, 5 Sep 2024 17:18:39 +0800 Subject: [PATCH 09/23] fix: gitlab mr comments won't be converted to domain layer till next run --- backend/plugins/gitlab/tasks/mr_note_convertor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/plugins/gitlab/tasks/mr_note_convertor.go b/backend/plugins/gitlab/tasks/mr_note_convertor.go index 8c600851e17..d1291d464da 100644 --- a/backend/plugins/gitlab/tasks/mr_note_convertor.go +++ b/backend/plugins/gitlab/tasks/mr_note_convertor.go @@ -38,7 +38,7 @@ var ConvertMrCommentMeta = plugin.SubTaskMeta{ EnabledByDefault: true, Description: "Add domain layer Comment according to GitlabMrComment", DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW}, - Dependencies: []*plugin.SubTaskMeta{&ConvertApiMergeRequestsMeta}, + Dependencies: []*plugin.SubTaskMeta{&ConvertApiMergeRequestsMeta, &ExtractApiMrNotesMeta}, } func ConvertMergeRequestNote(subtaskCtx plugin.SubTaskContext) errors.Error { From 191f2e6657a5cbf99431ba15e62f8beb940aadc8 Mon Sep 17 00:00:00 2001 From: Klesh Wong Date: Thu, 5 Sep 2024 17:19:55 +0800 Subject: [PATCH 10/23] feat: improve gitlab MR comments/commits collection performance --- backend/plugins/gitlab/tasks/shared.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/plugins/gitlab/tasks/shared.go b/backend/plugins/gitlab/tasks/shared.go index 976052968ff..59ce8b921ef 100644 --- a/backend/plugins/gitlab/tasks/shared.go +++ b/backend/plugins/gitlab/tasks/shared.go @@ -187,9 +187,10 @@ func GetMergeRequestsIterator(taskCtx plugin.SubTaskContext, apiCollector *helpe clauses := []dal.Clause{ dal.Select("gmr.gitlab_id, gmr.iid"), dal.From("_tool_gitlab_merge_requests gmr"), + // collect only openning merge request's notes and commits to speed up the process dal.Where( - `gmr.project_id = ? and gmr.connection_id = ?`, - data.Options.ProjectId, data.Options.ConnectionId, + `gmr.project_id = ? and gmr.connection_id = ? AND state = ?`, + data.Options.ProjectId, data.Options.ConnectionId, "opened", ), } if apiCollector != nil { From 3307ad979e0e7933bf6f535e7f2c45aa1f3102af Mon Sep 17 00:00:00 2001 From: Klesh Wong Date: Thu, 5 Sep 2024 17:27:09 +0800 Subject: [PATCH 11/23] fix: gitlab issues/mrs child records are not deleted --- .../api/data_convertor_stateful.go | 23 +++++++++++++++--- .../plugins/gitlab/tasks/issue_convertor.go | 11 +++++++++ .../plugins/gitlab/tasks/issue_extractor.go | 22 ++++++++++++++--- backend/plugins/gitlab/tasks/mr_convertor.go | 19 +++++++++++++++ backend/plugins/gitlab/tasks/mr_extractor.go | 24 +++++++++++++++++++ 5 files changed, 93 insertions(+), 6 deletions(-) diff --git a/backend/helpers/pluginhelper/api/data_convertor_stateful.go b/backend/helpers/pluginhelper/api/data_convertor_stateful.go index 058a1fa3255..395c021fb22 100644 --- a/backend/helpers/pluginhelper/api/data_convertor_stateful.go +++ b/backend/helpers/pluginhelper/api/data_convertor_stateful.go @@ -27,9 +27,10 @@ import ( type StatefulDataConverterArgs[InputType any] struct { *SubtaskCommonArgs - Input func(*SubtaskStateManager) (dal.Rows, errors.Error) - Convert func(row *InputType) ([]any, errors.Error) - BatchSize int + Input func(*SubtaskStateManager) (dal.Rows, errors.Error) + BeforeConvert func(issue *InputType, stateManager *SubtaskStateManager) errors.Error + Convert func(row *InputType) ([]any, errors.Error) + BatchSize int } // StatefulDataConverter is a struct that manages the stateful data conversion process. @@ -78,6 +79,15 @@ type StatefulDataConverterArgs[InputType any] struct { // } // return db.Cursor(clauses...) // }, +// BeforeConvert: func(jiraIssue *models.GitlabMergeRequest, stateManager *api.SubtaskStateManager) errors.Error { +// // Delete child records +// issueId := issueIdGen.Generate(data.Options.ConnectionId, jiraIssue.IssueId) +// if err := db.Delete(&ticket.IssueAssignee{}, dal.Where("issue_id = ?", issueId)); err != nil { +// return err +// } +// ... +// return nil +// }, // Convert: func(jiraIssue *models.JiraIssue) ([]interface{}, errors.Error) { // }, // }) @@ -146,6 +156,13 @@ func (converter *StatefulDataConverter[InputType]) Execute() errors.Error { return errors.Default.Wrap(err, "error fetching rows") } + if converter.BeforeConvert != nil { + err = converter.BeforeConvert(inputRow, converter.SubtaskStateManager) + if err != nil { + return err + } + } + results, err := converter.Convert(inputRow) if err != nil { return errors.Default.Wrap(err, "error calling Converter plugin implementation") diff --git a/backend/plugins/gitlab/tasks/issue_convertor.go b/backend/plugins/gitlab/tasks/issue_convertor.go index 236f899de6e..fffcc4119e8 100644 --- a/backend/plugins/gitlab/tasks/issue_convertor.go +++ b/backend/plugins/gitlab/tasks/issue_convertor.go @@ -68,6 +68,16 @@ func ConvertIssues(subtaskCtx plugin.SubTaskContext) errors.Error { } return db.Cursor(clauses...) }, + BeforeConvert: func(issue *models.GitlabIssue, stateManager *api.SubtaskStateManager) errors.Error { + issueId := issueIdGen.Generate(data.Options.ConnectionId, issue.GitlabId) + if err := db.Delete(&ticket.IssueAssignee{}, dal.Where("issue_id = ?", issueId)); err != nil { + return err + } + if err := db.Delete(&ticket.IssueLabel{}, dal.Where("issue_id = ?", issueId)); err != nil { + return err + } + return nil + }, Convert: func(issue *models.GitlabIssue) ([]interface{}, errors.Error) { domainIssue := &ticket.Issue{ DomainEntity: domainlayer.DomainEntity{Id: issueIdGen.Generate(data.Options.ConnectionId, issue.GitlabId)}, @@ -105,6 +115,7 @@ func ConvertIssues(subtaskCtx plugin.SubTaskContext) errors.Error { BoardId: boardIdGen.Generate(data.Options.ConnectionId, projectId), IssueId: domainIssue.Id, } + return []interface{}{ domainIssue, boardIssue, diff --git a/backend/plugins/gitlab/tasks/issue_extractor.go b/backend/plugins/gitlab/tasks/issue_extractor.go index 9685339af74..179b28e2d40 100644 --- a/backend/plugins/gitlab/tasks/issue_extractor.go +++ b/backend/plugins/gitlab/tasks/issue_extractor.go @@ -21,6 +21,7 @@ import ( "encoding/json" "regexp" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/common" "github.com/apache/incubator-devlake/core/plugin" @@ -133,9 +134,10 @@ type IssuesResponse struct { } } -func ExtractApiIssues(taskCtx plugin.SubTaskContext) errors.Error { - subtaskCommonArgs, data := CreateSubtaskCommonArgs(taskCtx, RAW_ISSUE_TABLE) +func ExtractApiIssues(subtaskCtx plugin.SubTaskContext) errors.Error { + subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_ISSUE_TABLE) + db := subtaskCtx.GetDal() config := data.Options.ScopeConfig var issueSeverityRegex *regexp.Regexp var issueComponentRegex *regexp.Regexp @@ -186,11 +188,18 @@ func ExtractApiIssues(taskCtx plugin.SubTaskContext) errors.Error { if err != nil { return nil, err } + err = db.Delete( + &models.GitlabIssueLabel{}, + dal.Where("connection_id = ? AND issue_id = ?", data.Options.ConnectionId, gitlabIssue.GitlabId), + ) + if err != nil { + return nil, err + } for _, label := range body.Labels { results = append(results, &models.GitlabIssueLabel{ + ConnectionId: data.Options.ConnectionId, IssueId: gitlabIssue.GitlabId, LabelName: label, - ConnectionId: data.Options.ConnectionId, }) if issueSeverityRegex != nil && issueSeverityRegex.MatchString(label) { gitlabIssue.Severity = label @@ -213,6 +222,13 @@ func ExtractApiIssues(taskCtx plugin.SubTaskContext) errors.Error { } results = append(results, gitlabIssue) + err = db.Delete( + &models.GitlabIssueAssignee{}, + dal.Where("connection_id = ? AND gitlab_id = ?", data.Options.ConnectionId, gitlabIssue.GitlabId), + ) + if err != nil { + return nil, err + } for _, v := range body.Assignees { assignee := &models.GitlabAccount{ ConnectionId: data.Options.ConnectionId, diff --git a/backend/plugins/gitlab/tasks/mr_convertor.go b/backend/plugins/gitlab/tasks/mr_convertor.go index b6b11e498d7..22c6b73c2e1 100644 --- a/backend/plugins/gitlab/tasks/mr_convertor.go +++ b/backend/plugins/gitlab/tasks/mr_convertor.go @@ -64,6 +64,25 @@ func ConvertApiMergeRequests(subtaskCtx plugin.SubTaskContext) errors.Error { } return db.Cursor(clauses...) }, + BeforeConvert: func(gitlabMr *models.GitlabMergeRequest, stateManager *api.SubtaskStateManager) errors.Error { + mrId := domainMrIdGenerator.Generate(data.Options.ConnectionId, gitlabMr.GitlabId) + if err := db.Delete(&code.PullRequestAssignee{}, dal.Where("pull_request_id = ?", mrId)); err != nil { + return err + } + if err := db.Delete(&code.PullRequestReviewer{}, dal.Where("pull_request_id = ?", mrId)); err != nil { + return err + } + if err := db.Delete(&code.PullRequestComment{}, dal.Where("pull_request_id = ?", mrId)); err != nil { + return err + } + if err := db.Delete(&code.PullRequestCommit{}, dal.Where("pull_request_id = ?", mrId)); err != nil { + return err + } + if err := db.Delete(&code.PullRequestLabel{}, dal.Where("pull_request_id = ?", mrId)); err != nil { + return err + } + return nil + }, Convert: func(gitlabMr *models.GitlabMergeRequest) ([]interface{}, errors.Error) { domainPr := &code.PullRequest{ DomainEntity: domainlayer.DomainEntity{ diff --git a/backend/plugins/gitlab/tasks/mr_extractor.go b/backend/plugins/gitlab/tasks/mr_extractor.go index f2508052719..9a56fd010b2 100644 --- a/backend/plugins/gitlab/tasks/mr_extractor.go +++ b/backend/plugins/gitlab/tasks/mr_extractor.go @@ -22,6 +22,7 @@ import ( "regexp" "strings" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/common" "github.com/apache/incubator-devlake/core/plugin" @@ -98,6 +99,8 @@ var ExtractApiMergeRequestsMeta = plugin.SubTaskMeta{ func ExtractApiMergeRequests(subtaskCtx plugin.SubTaskContext) errors.Error { subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_MERGE_REQUEST_TABLE) + + db := subtaskCtx.GetDal() config := data.Options.ScopeConfig var labelTypeRegex *regexp.Regexp var labelComponentRegex *regexp.Regexp @@ -149,6 +152,13 @@ func ExtractApiMergeRequests(subtaskCtx plugin.SubTaskContext) errors.Error { results := make([]interface{}, 0, len(mr.Reviewers)+len(mr.Labels)+1) gitlabMergeRequest.ConnectionId = data.Options.ConnectionId results = append(results, gitlabMergeRequest) + err = db.Delete( + &models.GitlabMrLabel{}, + dal.Where("connection_id = ? AND mr_id = ?", data.Options.ConnectionId, gitlabMergeRequest.GitlabId), + ) + if err != nil { + return nil, err + } for _, label := range mr.Labels { results = append(results, &models.GitlabMrLabel{ MrId: gitlabMergeRequest.GitlabId, @@ -164,6 +174,13 @@ func ExtractApiMergeRequests(subtaskCtx plugin.SubTaskContext) errors.Error { gitlabMergeRequest.Component = label } } + err = db.Delete( + &models.GitlabReviewer{}, + dal.Where("connection_id = ? AND merge_request_id = ?", data.Options.ConnectionId, gitlabMergeRequest.GitlabId), + ) + if err != nil { + return nil, err + } for _, reviewer := range mr.Reviewers { gitlabReviewer := &models.GitlabReviewer{ ConnectionId: data.Options.ConnectionId, @@ -178,6 +195,13 @@ func ExtractApiMergeRequests(subtaskCtx plugin.SubTaskContext) errors.Error { } results = append(results, gitlabReviewer) } + err = db.Delete( + &models.GitlabAssignee{}, + dal.Where("connection_id = ? AND merge_request_id = ?", data.Options.ConnectionId, gitlabMergeRequest.GitlabId), + ) + if err != nil { + return nil, err + } for _, assignee := range mr.Assignees { gitlabAssignee := &models.GitlabAssignee{ ConnectionId: data.Options.ConnectionId, From 637b5c56867d861dc343afeec04b658f2d3ee173 Mon Sep 17 00:00:00 2001 From: Klesh Wong Date: Thu, 5 Sep 2024 17:34:39 +0800 Subject: [PATCH 12/23] docs: update stateful extractor doc --- backend/helpers/pluginhelper/api/api_extractor_stateful.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/helpers/pluginhelper/api/api_extractor_stateful.go b/backend/helpers/pluginhelper/api/api_extractor_stateful.go index 00e2764b4c3..951383ff1c4 100644 --- a/backend/helpers/pluginhelper/api/api_extractor_stateful.go +++ b/backend/helpers/pluginhelper/api/api_extractor_stateful.go @@ -55,6 +55,10 @@ type StatefulApiExtractorArgs struct { // // It is also recommended that the configuration includes only the necessary fields used by the extractor. // ..}, // Extract: func(row *api.RawData) ([]interface{}, errors.Error) { +// jiraIssue := &models.JiraIssue{} +// json.Unmarshal([]byte(row.Data), jiraIssue) +// // It is important to delete the existing child-records if any +// db.Delete(&models.JiraIssueLabel{}, dal.Where("connection_id ? = ? AND issue_id = ?", issue.ConnectionId, issue.IssueId)) // return extractIssues(data, config, row, userFieldMap) // }, // }) From 5b9da49e1de1394891fc75a2d5ea4cfdbdef5a7c Mon Sep 17 00:00:00 2001 From: Klesh Wong Date: Fri, 6 Sep 2024 11:31:57 +0800 Subject: [PATCH 13/23] fix: gitlab mr detail test --- backend/plugins/gitlab/e2e/mr_detail_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/plugins/gitlab/e2e/mr_detail_test.go b/backend/plugins/gitlab/e2e/mr_detail_test.go index c9235667634..02d1e550134 100644 --- a/backend/plugins/gitlab/e2e/mr_detail_test.go +++ b/backend/plugins/gitlab/e2e/mr_detail_test.go @@ -41,6 +41,8 @@ func TestGitlabMrDetailDataFlow(t *testing.T) { }, } // import raw data table + dataflowTester.FlushTabler(&code.PullRequestAssignee{}) + dataflowTester.FlushTabler(&code.PullRequestReviewer{}) dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_gitlab_api_merge_requests.csv", "_raw_gitlab_api_merge_request_details") From a403d48dc4b858eb6a73c5c7274571bd22a7b47a Mon Sep 17 00:00:00 2001 From: Klesh Wong Date: Fri, 6 Sep 2024 15:13:36 +0800 Subject: [PATCH 14/23] refactor: unify stateful extractor and convertor helper --- .../api/api_extractor_stateful.go | 39 ++++++++----- .../gitlab/tasks/deployment_extractor.go | 10 +--- .../plugins/gitlab/tasks/issue_extractor.go | 42 +++++++------- backend/plugins/gitlab/tasks/job_extractor.go | 12 +--- .../gitlab/tasks/mr_commit_extractor.go | 9 +-- .../gitlab/tasks/mr_detail_extractor.go | 13 ++--- backend/plugins/gitlab/tasks/mr_extractor.go | 57 +++++++++++++++---- .../plugins/gitlab/tasks/mr_note_extractor.go | 12 +--- .../gitlab/tasks/pipeline_detail_extractor.go | 13 +---- .../gitlab/tasks/pipeline_extractor.go | 13 +---- .../gitlab/tasks/trigger_job_extractor.go | 12 +--- backend/plugins/jira/tasks/epic_extractor.go | 10 +++- .../jira/tasks/issue_changelog_extractor.go | 9 +-- backend/plugins/jira/tasks/issue_convertor.go | 11 ++++ backend/plugins/jira/tasks/issue_extractor.go | 42 ++++++++++---- 15 files changed, 164 insertions(+), 140 deletions(-) diff --git a/backend/helpers/pluginhelper/api/api_extractor_stateful.go b/backend/helpers/pluginhelper/api/api_extractor_stateful.go index 951383ff1c4..5225d8efab2 100644 --- a/backend/helpers/pluginhelper/api/api_extractor_stateful.go +++ b/backend/helpers/pluginhelper/api/api_extractor_stateful.go @@ -18,6 +18,7 @@ limitations under the License. package api import ( + "encoding/json" "reflect" "github.com/apache/incubator-devlake/core/dal" @@ -27,9 +28,10 @@ import ( ) // StatefulApiExtractorArgs is a struct that contains the arguments for a stateful api extractor -type StatefulApiExtractorArgs struct { +type StatefulApiExtractorArgs[InputType any] struct { *SubtaskCommonArgs - Extract func(row *RawData) ([]any, errors.Error) + BeforeExtract func(issue *InputType, stateManager *SubtaskStateManager) errors.Error + Extract func(body *InputType, row *RawData) ([]any, errors.Error) } // StatefulApiExtractor is a struct that manages the stateful API extraction process. @@ -41,7 +43,7 @@ type StatefulApiExtractorArgs struct { // // Example: // -// extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs{ +// extractor, err := api.NewStatefulApiExtractor[apiv2models.Issue](&api.StatefulApiExtractorArgs[apiv2models.Issue]{ // SubtaskCommonArgs: &api.SubtaskCommonArgs{ // SubTaskContext: subtaskCtx, // Table: RAW_ISSUE_TABLE, @@ -54,9 +56,7 @@ type StatefulApiExtractorArgs struct { // // Ensure that the configuration is serializable and contains only public fields. // // It is also recommended that the configuration includes only the necessary fields used by the extractor. // ..}, -// Extract: func(row *api.RawData) ([]interface{}, errors.Error) { -// jiraIssue := &models.JiraIssue{} -// json.Unmarshal([]byte(row.Data), jiraIssue) +// Extract: func(apiIssue *apiv2models.Issue, row *api.RawData) ([]interface{}, errors.Error) { // // It is important to delete the existing child-records if any // db.Delete(&models.JiraIssueLabel{}, dal.Where("connection_id ? = ? AND issue_id = ?", issue.ConnectionId, issue.IssueId)) // return extractIssues(data, config, row, userFieldMap) @@ -68,25 +68,25 @@ type StatefulApiExtractorArgs struct { // } // // return extractor.Execute() -type StatefulApiExtractor struct { - *StatefulApiExtractorArgs +type StatefulApiExtractor[InputType any] struct { + *StatefulApiExtractorArgs[InputType] *SubtaskStateManager } // NewStatefulApiExtractor creates a new StatefulApiExtractor -func NewStatefulApiExtractor(args *StatefulApiExtractorArgs) (*StatefulApiExtractor, errors.Error) { +func NewStatefulApiExtractor[InputType any](args *StatefulApiExtractorArgs[InputType]) (*StatefulApiExtractor[InputType], errors.Error) { stateManager, err := NewSubtaskStateManager(args.SubtaskCommonArgs) if err != nil { return nil, err } - return &StatefulApiExtractor{ + return &StatefulApiExtractor[InputType]{ StatefulApiExtractorArgs: args, SubtaskStateManager: stateManager, }, nil } // Execute sub-task -func (extractor *StatefulApiExtractor) Execute() errors.Error { +func (extractor *StatefulApiExtractor[InputType]) Execute() errors.Error { // load data from database db := extractor.GetDal() logger := extractor.GetLogger() @@ -139,7 +139,20 @@ func (extractor *StatefulApiExtractor) Execute() errors.Error { return errors.Default.Wrap(err, "error fetching row") } - results, err := extractor.Extract(row) + body := new(InputType) + err = errors.Convert(json.Unmarshal(row.Data, body)) + if err != nil { + return err + } + + if extractor.BeforeExtract != nil { + err = extractor.BeforeExtract(body, extractor.SubtaskStateManager) + if err != nil { + return err + } + } + + results, err := extractor.Extract(body, row) if err != nil { return errors.Default.Wrap(err, "error calling plugin Extract implementation") } @@ -173,4 +186,4 @@ func (extractor *StatefulApiExtractor) Execute() errors.Error { return extractor.SubtaskStateManager.Close() } -var _ plugin.SubTask = (*StatefulApiExtractor)(nil) +var _ plugin.SubTask = (*StatefulApiExtractor[any])(nil) diff --git a/backend/plugins/gitlab/tasks/deployment_extractor.go b/backend/plugins/gitlab/tasks/deployment_extractor.go index 274714b2df4..07fbb0cb514 100644 --- a/backend/plugins/gitlab/tasks/deployment_extractor.go +++ b/backend/plugins/gitlab/tasks/deployment_extractor.go @@ -18,7 +18,6 @@ limitations under the License. package tasks import ( - "encoding/json" "time" "github.com/apache/incubator-devlake/core/errors" @@ -46,14 +45,9 @@ var ExtractDeploymentMeta = plugin.SubTaskMeta{ func ExtractDeployment(taskCtx plugin.SubTaskContext) errors.Error { subtaskCommonArgs, data := CreateSubtaskCommonArgs(taskCtx, RAW_DEPLOYMENT) - extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs{ + extractor, err := api.NewStatefulApiExtractor[GitlabDeploymentResp](&api.StatefulApiExtractorArgs[GitlabDeploymentResp]{ SubtaskCommonArgs: subtaskCommonArgs, - Extract: func(row *api.RawData) ([]interface{}, errors.Error) { - deploymentResp := &GitlabDeploymentResp{} - err := errors.Convert(json.Unmarshal(row.Data, deploymentResp)) - if err != nil { - return nil, err - } + Extract: func(deploymentResp *GitlabDeploymentResp, row *api.RawData) ([]interface{}, errors.Error) { gitlabDeployment := deploymentResp.toGitlabDeployment(data.Options.ConnectionId, data.Options.ProjectId) return []interface{}{ gitlabDeployment, diff --git a/backend/plugins/gitlab/tasks/issue_extractor.go b/backend/plugins/gitlab/tasks/issue_extractor.go index 179b28e2d40..502eae82b02 100644 --- a/backend/plugins/gitlab/tasks/issue_extractor.go +++ b/backend/plugins/gitlab/tasks/issue_extractor.go @@ -18,7 +18,6 @@ limitations under the License. package tasks import ( - "encoding/json" "regexp" "github.com/apache/incubator-devlake/core/dal" @@ -170,15 +169,28 @@ func ExtractApiIssues(subtaskCtx plugin.SubTaskContext) errors.Error { "issueComponent": issueComponent, "issuePriorityRegex": issuePriorityRegex, } - extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs{ + extractor, err := api.NewStatefulApiExtractor[IssuesResponse](&api.StatefulApiExtractorArgs[IssuesResponse]{ SubtaskCommonArgs: subtaskCommonArgs, - Extract: func(row *api.RawData) ([]interface{}, errors.Error) { - body := &IssuesResponse{} - err := errors.Convert(json.Unmarshal(row.Data, body)) - if err != nil { - return nil, err + BeforeExtract: func(body *IssuesResponse, stateManager *api.SubtaskStateManager) errors.Error { + if stateManager.IsIncremental() { + err := db.Delete( + &models.GitlabIssueLabel{}, + dal.Where("connection_id = ? AND issue_id = ?", data.Options.ConnectionId, body.Id), + ) + if err != nil { + return err + } + err = db.Delete( + &models.GitlabIssueAssignee{}, + dal.Where("connection_id = ? AND gitlab_id = ?", data.Options.ConnectionId, body.Id), + ) + if err != nil { + return err + } } - + return nil + }, + Extract: func(body *IssuesResponse, row *api.RawData) ([]interface{}, errors.Error) { if body.ProjectId == 0 { return nil, nil } @@ -188,13 +200,6 @@ func ExtractApiIssues(subtaskCtx plugin.SubTaskContext) errors.Error { if err != nil { return nil, err } - err = db.Delete( - &models.GitlabIssueLabel{}, - dal.Where("connection_id = ? AND issue_id = ?", data.Options.ConnectionId, gitlabIssue.GitlabId), - ) - if err != nil { - return nil, err - } for _, label := range body.Labels { results = append(results, &models.GitlabIssueLabel{ ConnectionId: data.Options.ConnectionId, @@ -222,13 +227,6 @@ func ExtractApiIssues(subtaskCtx plugin.SubTaskContext) errors.Error { } results = append(results, gitlabIssue) - err = db.Delete( - &models.GitlabIssueAssignee{}, - dal.Where("connection_id = ? AND gitlab_id = ?", data.Options.ConnectionId, gitlabIssue.GitlabId), - ) - if err != nil { - return nil, err - } for _, v := range body.Assignees { assignee := &models.GitlabAccount{ ConnectionId: data.Options.ConnectionId, diff --git a/backend/plugins/gitlab/tasks/job_extractor.go b/backend/plugins/gitlab/tasks/job_extractor.go index 65bfb33aabf..aec601bd4f1 100644 --- a/backend/plugins/gitlab/tasks/job_extractor.go +++ b/backend/plugins/gitlab/tasks/job_extractor.go @@ -18,8 +18,6 @@ limitations under the License. package tasks import ( - "encoding/json" - "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/common" "github.com/apache/incubator-devlake/core/plugin" @@ -62,16 +60,10 @@ var ExtractApiJobsMeta = plugin.SubTaskMeta{ func ExtractApiJobs(subtaskCtx plugin.SubTaskContext) errors.Error { subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_JOB_TABLE) - extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs{ + extractor, err := api.NewStatefulApiExtractor[ApiJob](&api.StatefulApiExtractorArgs[ApiJob]{ SubtaskCommonArgs: subtaskCommonArgs, - Extract: func(row *api.RawData) ([]interface{}, errors.Error) { + Extract: func(gitlabApiJob *ApiJob, row *api.RawData) ([]interface{}, errors.Error) { // create gitlab commit - gitlabApiJob := &ApiJob{} - err := errors.Convert(json.Unmarshal(row.Data, gitlabApiJob)) - if err != nil { - return nil, err - } - gitlabPipeline, err := convertJob(gitlabApiJob, data.Options.ProjectId) if err != nil { return nil, err diff --git a/backend/plugins/gitlab/tasks/mr_commit_extractor.go b/backend/plugins/gitlab/tasks/mr_commit_extractor.go index 12f0d96be98..b0568c8c2b6 100644 --- a/backend/plugins/gitlab/tasks/mr_commit_extractor.go +++ b/backend/plugins/gitlab/tasks/mr_commit_extractor.go @@ -43,15 +43,10 @@ var ExtractApiMrCommitsMeta = plugin.SubTaskMeta{ func ExtractApiMergeRequestsCommits(subtaskCtx plugin.SubTaskContext) errors.Error { subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_MERGE_REQUEST_COMMITS_TABLE) - extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs{ + extractor, err := api.NewStatefulApiExtractor[GitlabApiCommit](&api.StatefulApiExtractorArgs[GitlabApiCommit]{ SubtaskCommonArgs: subtaskCommonArgs, - Extract: func(row *api.RawData) ([]interface{}, errors.Error) { + Extract: func(gitlabApiCommit *GitlabApiCommit, row *api.RawData) ([]interface{}, errors.Error) { // create gitlab commit - gitlabApiCommit := &GitlabApiCommit{} - err := errors.Convert(json.Unmarshal(row.Data, gitlabApiCommit)) - if err != nil { - return nil, err - } gitlabCommit, err := ConvertCommit(gitlabApiCommit) if err != nil { return nil, err diff --git a/backend/plugins/gitlab/tasks/mr_detail_extractor.go b/backend/plugins/gitlab/tasks/mr_detail_extractor.go index 142cf2de875..cf23cb48c14 100644 --- a/backend/plugins/gitlab/tasks/mr_detail_extractor.go +++ b/backend/plugins/gitlab/tasks/mr_detail_extractor.go @@ -18,7 +18,6 @@ limitations under the License. package tasks import ( - "encoding/json" "regexp" "github.com/apache/incubator-devlake/core/errors" @@ -42,6 +41,7 @@ var ExtractApiMergeRequestDetailsMeta = plugin.SubTaskMeta{ func ExtractApiMergeRequestDetails(subtaskCtx plugin.SubTaskContext) errors.Error { subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_MERGE_REQUEST_DETAIL_TABLE) + db := subtaskCtx.GetDal() config := data.Options.ScopeConfig var labelTypeRegex *regexp.Regexp var labelComponentRegex *regexp.Regexp @@ -64,15 +64,10 @@ func ExtractApiMergeRequestDetails(subtaskCtx plugin.SubTaskContext) errors.Erro "prType": prType, "prComponent": prComponent, } - extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs{ + extractor, err := api.NewStatefulApiExtractor[MergeRequestRes](&api.StatefulApiExtractorArgs[MergeRequestRes]{ SubtaskCommonArgs: subtaskCommonArgs, - Extract: func(row *api.RawData) ([]interface{}, errors.Error) { - mr := &MergeRequestRes{} - err := errors.Convert(json.Unmarshal(row.Data, mr)) - if err != nil { - return nil, err - } - + BeforeExtract: beforeExtractMr(db, data), + Extract: func(mr *MergeRequestRes, row *api.RawData) ([]interface{}, errors.Error) { gitlabMergeRequest, err := convertMergeRequest(mr) if err != nil { return nil, err diff --git a/backend/plugins/gitlab/tasks/mr_extractor.go b/backend/plugins/gitlab/tasks/mr_extractor.go index 9a56fd010b2..c4647af0866 100644 --- a/backend/plugins/gitlab/tasks/mr_extractor.go +++ b/backend/plugins/gitlab/tasks/mr_extractor.go @@ -18,7 +18,6 @@ limitations under the License. package tasks import ( - "encoding/json" "regexp" "strings" @@ -126,16 +125,10 @@ func ExtractApiMergeRequests(subtaskCtx plugin.SubTaskContext) errors.Error { "prComponent": prComponent, } - extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs{ + extractor, err := api.NewStatefulApiExtractor[MergeRequestRes](&api.StatefulApiExtractorArgs[MergeRequestRes]{ SubtaskCommonArgs: subtaskCommonArgs, - Extract: func(row *api.RawData) ([]interface{}, errors.Error) { - mr := &MergeRequestRes{} - s := string(row.Data) - err := errors.Convert(json.Unmarshal(row.Data, mr)) - if err != nil { - return nil, err - } - + BeforeExtract: beforeExtractMr(db, data), + Extract: func(mr *MergeRequestRes, row *api.RawData) ([]interface{}, errors.Error) { gitlabMergeRequest, err := convertMergeRequest(mr) if err != nil { return nil, err @@ -143,6 +136,7 @@ func ExtractApiMergeRequests(subtaskCtx plugin.SubTaskContext) errors.Error { // if we can not find merged_at and closed_at info in the detail // we need get detail for gitlab v11 + s := string(row.Data) if !strings.Contains(s, "\"merged_at\":") { if !strings.Contains(s, "\"closed_at\":") { gitlabMergeRequest.IsDetailRequired = true @@ -254,3 +248,46 @@ func convertMergeRequest(mr *MergeRequestRes) (*models.GitlabMergeRequest, error } return gitlabMergeRequest, nil } + +func beforeExtractMr(db dal.Dal, data *GitlabTaskData) func(mr *MergeRequestRes, stateManager *api.SubtaskStateManager) errors.Error { + return func(mr *MergeRequestRes, stateManager *api.SubtaskStateManager) errors.Error { + if stateManager.IsIncremental() { + err := db.Delete( + &models.GitlabMrLabel{}, + dal.Where("connection_id = ? AND mr_id = ?", data.Options.ConnectionId, mr.GitlabId), + ) + if err != nil { + return err + } + err = db.Delete( + &models.GitlabIssueAssignee{}, + dal.Where("connection_id = ? AND merge_request_id = ?", data.Options.ConnectionId, mr.GitlabId), + ) + if err != nil { + return err + } + err = db.Delete( + &models.GitlabAssignee{}, + dal.Where("connection_id = ? AND merge_request_id = ?", data.Options.ConnectionId, mr.GitlabId), + ) + if err != nil { + return err + } + err = db.Delete( + &models.GitlabMrCommit{}, + dal.Where("connection_id = ? AND merge_request_id = ?", data.Options.ConnectionId, mr.GitlabId), + ) + if err != nil { + return err + } + err = db.Delete( + &models.GitlabMrComment{}, + dal.Where("connection_id = ? AND merge_request_id = ?", data.Options.ConnectionId, mr.GitlabId), + ) + if err != nil { + return err + } + } + return nil + } +} diff --git a/backend/plugins/gitlab/tasks/mr_note_extractor.go b/backend/plugins/gitlab/tasks/mr_note_extractor.go index 218b741fc4e..9354ca86da3 100644 --- a/backend/plugins/gitlab/tasks/mr_note_extractor.go +++ b/backend/plugins/gitlab/tasks/mr_note_extractor.go @@ -18,8 +18,6 @@ limitations under the License. package tasks import ( - "encoding/json" - "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/common" "github.com/apache/incubator-devlake/core/plugin" @@ -60,15 +58,9 @@ var ExtractApiMrNotesMeta = plugin.SubTaskMeta{ func ExtractApiMergeRequestsNotes(subtaskCtx plugin.SubTaskContext) errors.Error { subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_MERGE_REQUEST_NOTES_TABLE) - extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs{ + extractor, err := api.NewStatefulApiExtractor[MergeRequestNote](&api.StatefulApiExtractorArgs[MergeRequestNote]{ SubtaskCommonArgs: subtaskCommonArgs, - Extract: func(row *api.RawData) ([]interface{}, errors.Error) { - mrNote := &MergeRequestNote{} - err := errors.Convert(json.Unmarshal(row.Data, mrNote)) - if err != nil { - return nil, err - } - + Extract: func(mrNote *MergeRequestNote, row *api.RawData) ([]interface{}, errors.Error) { toolMrNote, err := convertMergeRequestNote(mrNote) toolMrNote.ConnectionId = data.Options.ConnectionId if err != nil { diff --git a/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go b/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go index 49146450ed8..ab89cc4adb1 100644 --- a/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go +++ b/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go @@ -18,8 +18,6 @@ limitations under the License. package tasks import ( - "encoding/json" - "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/common" "github.com/apache/incubator-devlake/core/models/domainlayer/devops" @@ -44,16 +42,9 @@ var ExtractApiPipelineDetailsMeta = plugin.SubTaskMeta{ func ExtractApiPipelineDetails(subtaskCtx plugin.SubTaskContext) errors.Error { subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_PIPELINE_DETAILS_TABLE) - extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs{ + extractor, err := api.NewStatefulApiExtractor[ApiPipeline](&api.StatefulApiExtractorArgs[ApiPipeline]{ SubtaskCommonArgs: subtaskCommonArgs, - Extract: func(row *api.RawData) ([]interface{}, errors.Error) { - // create gitlab commit - gitlabApiPipeline := &ApiPipeline{} - err := errors.Convert(json.Unmarshal(row.Data, gitlabApiPipeline)) - if err != nil { - return nil, err - } - + Extract: func(gitlabApiPipeline *ApiPipeline, row *api.RawData) ([]interface{}, errors.Error) { gitlabPipeline := &models.GitlabPipeline{ GitlabId: gitlabApiPipeline.Id, ProjectId: data.Options.ProjectId, diff --git a/backend/plugins/gitlab/tasks/pipeline_extractor.go b/backend/plugins/gitlab/tasks/pipeline_extractor.go index c7069a56a5b..e5bba1cd6e2 100644 --- a/backend/plugins/gitlab/tasks/pipeline_extractor.go +++ b/backend/plugins/gitlab/tasks/pipeline_extractor.go @@ -18,8 +18,6 @@ limitations under the License. package tasks import ( - "encoding/json" - "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/common" "github.com/apache/incubator-devlake/core/plugin" @@ -72,16 +70,9 @@ var ExtractApiPipelinesMeta = plugin.SubTaskMeta{ func ExtractApiPipelines(subtaskCtx plugin.SubTaskContext) errors.Error { subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_PIPELINE_TABLE) - extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs{ + extractor, err := api.NewStatefulApiExtractor[ApiPipeline](&api.StatefulApiExtractorArgs[ApiPipeline]{ SubtaskCommonArgs: subtaskCommonArgs, - Extract: func(row *api.RawData) ([]interface{}, errors.Error) { - // create gitlab commit - gitlabApiPipeline := &ApiPipeline{} - err := errors.Convert(json.Unmarshal(row.Data, gitlabApiPipeline)) - if err != nil { - return nil, err - } - + Extract: func(gitlabApiPipeline *ApiPipeline, row *api.RawData) ([]interface{}, errors.Error) { pipelineProject := &models.GitlabPipelineProject{ ConnectionId: data.Options.ConnectionId, PipelineId: gitlabApiPipeline.Id, diff --git a/backend/plugins/gitlab/tasks/trigger_job_extractor.go b/backend/plugins/gitlab/tasks/trigger_job_extractor.go index 3ad3bb67aef..0da01208962 100644 --- a/backend/plugins/gitlab/tasks/trigger_job_extractor.go +++ b/backend/plugins/gitlab/tasks/trigger_job_extractor.go @@ -18,7 +18,6 @@ limitations under the License. package tasks import ( - "encoding/json" "time" "github.com/apache/incubator-devlake/core/errors" @@ -58,16 +57,9 @@ var ExtractApiTriggerJobsMeta = plugin.SubTaskMeta{ func ExtractApiTriggerJobs(subtaskCtx plugin.SubTaskContext) errors.Error { subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_TRIGGER_JOB_TABLE) - extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs{ + extractor, err := api.NewStatefulApiExtractor[ApiTriggerJob](&api.StatefulApiExtractorArgs[ApiTriggerJob]{ SubtaskCommonArgs: subtaskCommonArgs, - Extract: func(row *api.RawData) ([]interface{}, errors.Error) { - // create gitlab commit - gitlabApiTriggerJob := &ApiTriggerJob{} - err := errors.Convert(json.Unmarshal(row.Data, gitlabApiTriggerJob)) - if err != nil { - return nil, err - } - + Extract: func(gitlabApiTriggerJob *ApiTriggerJob, row *api.RawData) ([]interface{}, errors.Error) { gitlabPipeline, err := convertTriggerJob(gitlabApiTriggerJob, data.Options.ProjectId) if err != nil { return nil, err diff --git a/backend/plugins/jira/tasks/epic_extractor.go b/backend/plugins/jira/tasks/epic_extractor.go index cf1e7f28b89..89f2ef70cc6 100644 --- a/backend/plugins/jira/tasks/epic_extractor.go +++ b/backend/plugins/jira/tasks/epic_extractor.go @@ -18,12 +18,15 @@ limitations under the License. package tasks import ( + "encoding/json" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/log" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/jira/models" + "github.com/apache/incubator-devlake/plugins/jira/tasks/apiv2models" ) var _ plugin.SubTaskEntryPoint = ExtractEpics @@ -61,7 +64,12 @@ func ExtractEpics(taskCtx plugin.SubTaskContext) errors.Error { Table: RAW_EPIC_TABLE, }, Extract: func(row *api.RawData) ([]interface{}, errors.Error) { - return extractIssues(data, mappings, row, userFieldMap) + apiIssue := &apiv2models.Issue{} + err = errors.Convert(json.Unmarshal(row.Data, apiIssue)) + if err != nil { + return nil, err + } + return extractIssues(data, mappings, apiIssue, row, userFieldMap) }, }) if err != nil { diff --git a/backend/plugins/jira/tasks/issue_changelog_extractor.go b/backend/plugins/jira/tasks/issue_changelog_extractor.go index a49142e3f37..911185f651f 100644 --- a/backend/plugins/jira/tasks/issue_changelog_extractor.go +++ b/backend/plugins/jira/tasks/issue_changelog_extractor.go @@ -47,7 +47,7 @@ func ExtractIssueChangelogs(subtaskCtx plugin.SubTaskContext) errors.Error { if err != nil { return err } - extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs{ + extractor, err := api.NewStatefulApiExtractor[apiv2models.Changelog](&api.StatefulApiExtractorArgs[apiv2models.Changelog]{ SubtaskCommonArgs: &api.SubtaskCommonArgs{ SubTaskContext: subtaskCtx, Table: RAW_CHANGELOG_TABLE, @@ -56,18 +56,13 @@ func ExtractIssueChangelogs(subtaskCtx plugin.SubTaskContext) errors.Error { BoardId: data.Options.BoardId, }, }, - Extract: func(row *api.RawData) ([]interface{}, errors.Error) { + Extract: func(changelog *apiv2models.Changelog, row *api.RawData) ([]interface{}, errors.Error) { // process input var input apiv2models.Input err := errors.Convert(json.Unmarshal(row.Input, &input)) if err != nil { return nil, err } - var changelog apiv2models.Changelog - err = errors.Convert(json.Unmarshal(row.Data, &changelog)) - if err != nil { - return nil, err - } // prepare output var result []interface{} cl, user := changelog.ToToolLayer(connectionId, input.IssueId, &input.UpdateTime) diff --git a/backend/plugins/jira/tasks/issue_convertor.go b/backend/plugins/jira/tasks/issue_convertor.go index 1c7305758c1..a55c1642077 100644 --- a/backend/plugins/jira/tasks/issue_convertor.go +++ b/backend/plugins/jira/tasks/issue_convertor.go @@ -84,6 +84,17 @@ func ConvertIssues(subtaskCtx plugin.SubTaskContext) errors.Error { } return db.Cursor(clauses...) }, + // not needed for now due to jira assignee and label are converted in FullSync(Delete+Insert) manner + // BeforeConvert: func(jiraIssue *models.JiraIssue, stateManager *api.SubtaskStateManager) errors.Error { + // issueId := issueIdGen.Generate(data.Options.ConnectionId, jiraIssue.IssueId) + // if err := db.Delete(&ticket.IssueAssignee{}, dal.Where("issue_id = ?", issueId)); err != nil { + // return err + // } + // if err := db.Delete(&ticket.IssueLabel{}, dal.Where("issue_id = ?", issueId)); err != nil { + // return err + // } + // return nil + // }, Convert: func(jiraIssue *models.JiraIssue) ([]interface{}, errors.Error) { var result []interface{} issue := &ticket.Issue{ diff --git a/backend/plugins/jira/tasks/issue_extractor.go b/backend/plugins/jira/tasks/issue_extractor.go index 760b8c7cb4e..19b89bb7d95 100644 --- a/backend/plugins/jira/tasks/issue_extractor.go +++ b/backend/plugins/jira/tasks/issue_extractor.go @@ -18,7 +18,6 @@ limitations under the License. package tasks import ( - "encoding/json" "strconv" "strings" "time" @@ -62,7 +61,7 @@ func ExtractIssues(subtaskCtx plugin.SubTaskContext) errors.Error { if err != nil { return err } - extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs{ + extractor, err := api.NewStatefulApiExtractor[apiv2models.Issue](&api.StatefulApiExtractorArgs[apiv2models.Issue]{ SubtaskCommonArgs: &api.SubtaskCommonArgs{ SubTaskContext: subtaskCtx, Table: RAW_ISSUE_TABLE, @@ -72,8 +71,34 @@ func ExtractIssues(subtaskCtx plugin.SubTaskContext) errors.Error { }, SubtaskConfig: mappings, }, - Extract: func(row *api.RawData) ([]interface{}, errors.Error) { - return extractIssues(data, mappings, row, userFieldMap) + BeforeExtract: func(apiIssue *apiv2models.Issue, stateManager *api.SubtaskStateManager) errors.Error { + if stateManager.IsIncremental() { + err := db.Delete( + &models.JiraSprintIssue{}, + dal.Where("connection_id = ? AND issue_id = ?", data.Options.ConnectionId, apiIssue.ID), + ) + if err != nil { + return err + } + err = db.Delete( + &models.JiraIssueLabel{}, + dal.Where("connection_id = ? AND issue_id = ?", data.Options.ConnectionId, apiIssue.ID), + ) + if err != nil { + return err + } + err = db.Delete( + &models.JiraIssueRelationship{}, + dal.Where("connection_id = ? AND issue_id = ?", data.Options.ConnectionId, apiIssue.ID), + ) + if err != nil { + return err + } + } + return nil + }, + Extract: func(apiIssue *apiv2models.Issue, row *api.RawData) ([]interface{}, errors.Error) { + return extractIssues(data, mappings, apiIssue, row, userFieldMap) }, }) if err != nil { @@ -82,13 +107,8 @@ func ExtractIssues(subtaskCtx plugin.SubTaskContext) errors.Error { return extractor.Execute() } -func extractIssues(data *JiraTaskData, mappings *typeMappings, row *api.RawData, userFieldMaps map[string]struct{}) ([]interface{}, errors.Error) { - var apiIssue apiv2models.Issue - err := errors.Convert(json.Unmarshal(row.Data, &apiIssue)) - if err != nil { - return nil, err - } - err = apiIssue.SetAllFields(row.Data) +func extractIssues(data *JiraTaskData, mappings *typeMappings, apiIssue *apiv2models.Issue, row *api.RawData, userFieldMaps map[string]struct{}) ([]interface{}, errors.Error) { + err := apiIssue.SetAllFields(row.Data) if err != nil { return nil, err } From c640547e795cd86f3c358b080fd0ff1d9c000656 Mon Sep 17 00:00:00 2001 From: Klesh Wong Date: Fri, 6 Sep 2024 15:22:27 +0800 Subject: [PATCH 15/23] docs: update stateful extractor/convertor doc --- .../pluginhelper/api/api_extractor_stateful.go | 13 ++++++++++--- .../pluginhelper/api/data_convertor_stateful.go | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/backend/helpers/pluginhelper/api/api_extractor_stateful.go b/backend/helpers/pluginhelper/api/api_extractor_stateful.go index 5225d8efab2..1b27f43c579 100644 --- a/backend/helpers/pluginhelper/api/api_extractor_stateful.go +++ b/backend/helpers/pluginhelper/api/api_extractor_stateful.go @@ -56,10 +56,17 @@ type StatefulApiExtractorArgs[InputType any] struct { // // Ensure that the configuration is serializable and contains only public fields. // // It is also recommended that the configuration includes only the necessary fields used by the extractor. // ..}, +// BeforeExtract: func(body *IssuesResponse, stateManager *api.SubtaskStateManager) errors.Error { +// if stateManager.IsIncremental() { +// // It is important to delete all existing child-records under DiffSync Mode +// err := db.Delete( +// &models.JiraIssueLabel{}, +// dal.Where("connection_id = ? AND issue_id = ?", data.Options.ConnectionId, body.Id), +// ) +// } +// return nil +// }, // Extract: func(apiIssue *apiv2models.Issue, row *api.RawData) ([]interface{}, errors.Error) { -// // It is important to delete the existing child-records if any -// db.Delete(&models.JiraIssueLabel{}, dal.Where("connection_id ? = ? AND issue_id = ?", issue.ConnectionId, issue.IssueId)) -// return extractIssues(data, config, row, userFieldMap) // }, // }) // diff --git a/backend/helpers/pluginhelper/api/data_convertor_stateful.go b/backend/helpers/pluginhelper/api/data_convertor_stateful.go index 395c021fb22..c9f6fd95864 100644 --- a/backend/helpers/pluginhelper/api/data_convertor_stateful.go +++ b/backend/helpers/pluginhelper/api/data_convertor_stateful.go @@ -80,7 +80,7 @@ type StatefulDataConverterArgs[InputType any] struct { // return db.Cursor(clauses...) // }, // BeforeConvert: func(jiraIssue *models.GitlabMergeRequest, stateManager *api.SubtaskStateManager) errors.Error { -// // Delete child records +// // It is important to delete all existing child-records under DiffSync Mode // issueId := issueIdGen.Generate(data.Options.ConnectionId, jiraIssue.IssueId) // if err := db.Delete(&ticket.IssueAssignee{}, dal.Where("issue_id = ?", issueId)); err != nil { // return err From 51f9ccf0b06362819b8a39754067ec8502000ba4 Mon Sep 17 00:00:00 2001 From: Klesh Wong Date: Fri, 6 Sep 2024 15:39:29 +0800 Subject: [PATCH 16/23] refactor: remove useless type hint --- backend/helpers/pluginhelper/api/api_extractor_stateful.go | 2 +- backend/helpers/pluginhelper/api/data_convertor_stateful.go | 3 +-- backend/plugins/gitlab/tasks/deployment_convertor.go | 2 +- backend/plugins/gitlab/tasks/deployment_extractor.go | 2 +- backend/plugins/gitlab/tasks/issue_assignee_convertor.go | 2 +- backend/plugins/gitlab/tasks/issue_convertor.go | 2 +- backend/plugins/gitlab/tasks/issue_extractor.go | 2 +- backend/plugins/gitlab/tasks/issue_label_convertor.go | 2 +- backend/plugins/gitlab/tasks/job_convertor.go | 2 +- backend/plugins/gitlab/tasks/job_extractor.go | 2 +- backend/plugins/gitlab/tasks/mr_assignee_convertor.go | 2 +- backend/plugins/gitlab/tasks/mr_commit_convertor.go | 2 +- backend/plugins/gitlab/tasks/mr_commit_extractor.go | 2 +- backend/plugins/gitlab/tasks/mr_convertor.go | 2 +- backend/plugins/gitlab/tasks/mr_detail_extractor.go | 2 +- backend/plugins/gitlab/tasks/mr_extractor.go | 2 +- backend/plugins/gitlab/tasks/mr_label_convertor.go | 2 +- backend/plugins/gitlab/tasks/mr_note_convertor.go | 2 +- backend/plugins/gitlab/tasks/mr_note_extractor.go | 2 +- backend/plugins/gitlab/tasks/mr_reviewer_convertor.go | 2 +- backend/plugins/gitlab/tasks/pipeline_commit_convertor.go | 2 +- backend/plugins/gitlab/tasks/pipeline_detail_convertor.go | 2 +- backend/plugins/gitlab/tasks/pipeline_detail_extractor.go | 2 +- backend/plugins/gitlab/tasks/pipeline_extractor.go | 2 +- backend/plugins/gitlab/tasks/trigger_job_extractor.go | 2 +- backend/plugins/jira/tasks/issue_changelog_convertor.go | 2 +- backend/plugins/jira/tasks/issue_changelog_extractor.go | 2 +- backend/plugins/jira/tasks/issue_convertor.go | 2 +- backend/plugins/jira/tasks/issue_extractor.go | 2 +- 29 files changed, 29 insertions(+), 30 deletions(-) diff --git a/backend/helpers/pluginhelper/api/api_extractor_stateful.go b/backend/helpers/pluginhelper/api/api_extractor_stateful.go index 1b27f43c579..2c994afdb5f 100644 --- a/backend/helpers/pluginhelper/api/api_extractor_stateful.go +++ b/backend/helpers/pluginhelper/api/api_extractor_stateful.go @@ -43,7 +43,7 @@ type StatefulApiExtractorArgs[InputType any] struct { // // Example: // -// extractor, err := api.NewStatefulApiExtractor[apiv2models.Issue](&api.StatefulApiExtractorArgs[apiv2models.Issue]{ +// extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[apiv2models.Issue]{ // SubtaskCommonArgs: &api.SubtaskCommonArgs{ // SubTaskContext: subtaskCtx, // Table: RAW_ISSUE_TABLE, diff --git a/backend/helpers/pluginhelper/api/data_convertor_stateful.go b/backend/helpers/pluginhelper/api/data_convertor_stateful.go index c9f6fd95864..1c5292dc230 100644 --- a/backend/helpers/pluginhelper/api/data_convertor_stateful.go +++ b/backend/helpers/pluginhelper/api/data_convertor_stateful.go @@ -48,7 +48,7 @@ type StatefulDataConverterArgs[InputType any] struct { // // Example: // -// converter, err := api.NewStatefulDataConverter[models.JiraIssue](&api.StatefulDataConverterArgs[models.JiraIssue]{ +// converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.JiraIssue]{ // SubtaskCommonArgs: &api.SubtaskCommonArgs{ // SubTaskContext: subtaskCtx, // Table: RAW_ISSUE_TABLE, @@ -103,7 +103,6 @@ type StatefulDataConverter[InputType any] struct { } func NewStatefulDataConverter[ - OptType any, InputType any, ]( args *StatefulDataConverterArgs[InputType], diff --git a/backend/plugins/gitlab/tasks/deployment_convertor.go b/backend/plugins/gitlab/tasks/deployment_convertor.go index 1df80ec543e..b211867b4ae 100644 --- a/backend/plugins/gitlab/tasks/deployment_convertor.go +++ b/backend/plugins/gitlab/tasks/deployment_convertor.go @@ -62,7 +62,7 @@ func ConvertDeployment(subtaskCtx plugin.SubTaskContext) errors.Error { projectIdGen := didgen.NewDomainIdGenerator(&models.GitlabProject{}) idGen := didgen.NewDomainIdGenerator(&models.GitlabDeployment{}) - converter, err := api.NewStatefulDataConverter[models.GitlabDeployment](&api.StatefulDataConverterArgs[models.GitlabDeployment]{ + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GitlabDeployment]{ SubtaskCommonArgs: subtaskCommonArgs, Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { clauses := []dal.Clause{ diff --git a/backend/plugins/gitlab/tasks/deployment_extractor.go b/backend/plugins/gitlab/tasks/deployment_extractor.go index 07fbb0cb514..258471b01c4 100644 --- a/backend/plugins/gitlab/tasks/deployment_extractor.go +++ b/backend/plugins/gitlab/tasks/deployment_extractor.go @@ -45,7 +45,7 @@ var ExtractDeploymentMeta = plugin.SubTaskMeta{ func ExtractDeployment(taskCtx plugin.SubTaskContext) errors.Error { subtaskCommonArgs, data := CreateSubtaskCommonArgs(taskCtx, RAW_DEPLOYMENT) - extractor, err := api.NewStatefulApiExtractor[GitlabDeploymentResp](&api.StatefulApiExtractorArgs[GitlabDeploymentResp]{ + extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[GitlabDeploymentResp]{ SubtaskCommonArgs: subtaskCommonArgs, Extract: func(deploymentResp *GitlabDeploymentResp, row *api.RawData) ([]interface{}, errors.Error) { gitlabDeployment := deploymentResp.toGitlabDeployment(data.Options.ConnectionId, data.Options.ProjectId) diff --git a/backend/plugins/gitlab/tasks/issue_assignee_convertor.go b/backend/plugins/gitlab/tasks/issue_assignee_convertor.go index 65c3abeb138..2f063609ff2 100644 --- a/backend/plugins/gitlab/tasks/issue_assignee_convertor.go +++ b/backend/plugins/gitlab/tasks/issue_assignee_convertor.go @@ -48,7 +48,7 @@ func ConvertIssueAssignee(subtaskCtx plugin.SubTaskContext) errors.Error { issueIdGen := didgen.NewDomainIdGenerator(&models.GitlabIssue{}) accountIdGen := didgen.NewDomainIdGenerator(&models.GitlabAccount{}) - converter, err := api.NewStatefulDataConverter[models.GitlabIssueAssignee](&api.StatefulDataConverterArgs[models.GitlabIssueAssignee]{ + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GitlabIssueAssignee]{ SubtaskCommonArgs: subtaskCommonArgs, Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { clauses := []dal.Clause{ diff --git a/backend/plugins/gitlab/tasks/issue_convertor.go b/backend/plugins/gitlab/tasks/issue_convertor.go index fffcc4119e8..cefc18c1e59 100644 --- a/backend/plugins/gitlab/tasks/issue_convertor.go +++ b/backend/plugins/gitlab/tasks/issue_convertor.go @@ -53,7 +53,7 @@ func ConvertIssues(subtaskCtx plugin.SubTaskContext) errors.Error { accountIdGen := didgen.NewDomainIdGenerator(&models.GitlabAccount{}) boardIdGen := didgen.NewDomainIdGenerator(&models.GitlabProject{}) - converter, err := api.NewStatefulDataConverter[models.GitlabIssue](&api.StatefulDataConverterArgs[models.GitlabIssue]{ + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GitlabIssue]{ SubtaskCommonArgs: subtaskCommonArgs, Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { clauses := []dal.Clause{ diff --git a/backend/plugins/gitlab/tasks/issue_extractor.go b/backend/plugins/gitlab/tasks/issue_extractor.go index 502eae82b02..802d58a6074 100644 --- a/backend/plugins/gitlab/tasks/issue_extractor.go +++ b/backend/plugins/gitlab/tasks/issue_extractor.go @@ -169,7 +169,7 @@ func ExtractApiIssues(subtaskCtx plugin.SubTaskContext) errors.Error { "issueComponent": issueComponent, "issuePriorityRegex": issuePriorityRegex, } - extractor, err := api.NewStatefulApiExtractor[IssuesResponse](&api.StatefulApiExtractorArgs[IssuesResponse]{ + extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[IssuesResponse]{ SubtaskCommonArgs: subtaskCommonArgs, BeforeExtract: func(body *IssuesResponse, stateManager *api.SubtaskStateManager) errors.Error { if stateManager.IsIncremental() { diff --git a/backend/plugins/gitlab/tasks/issue_label_convertor.go b/backend/plugins/gitlab/tasks/issue_label_convertor.go index 3799335916f..fab745dda9e 100644 --- a/backend/plugins/gitlab/tasks/issue_label_convertor.go +++ b/backend/plugins/gitlab/tasks/issue_label_convertor.go @@ -47,7 +47,7 @@ func ConvertIssueLabels(subtaskCtx plugin.SubTaskContext) errors.Error { projectId := data.Options.ProjectId issueIdGen := didgen.NewDomainIdGenerator(&models.GitlabIssue{}) - converter, err := api.NewStatefulDataConverter[models.GitlabIssueLabel](&api.StatefulDataConverterArgs[models.GitlabIssueLabel]{ + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GitlabIssueLabel]{ SubtaskCommonArgs: subtaskCommonArgs, Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { clauses := []dal.Clause{ diff --git a/backend/plugins/gitlab/tasks/job_convertor.go b/backend/plugins/gitlab/tasks/job_convertor.go index acbec7df45d..91077844753 100644 --- a/backend/plugins/gitlab/tasks/job_convertor.go +++ b/backend/plugins/gitlab/tasks/job_convertor.go @@ -52,7 +52,7 @@ func ConvertJobs(subtaskCtx plugin.SubTaskContext) (err errors.Error) { jobIdGen := didgen.NewDomainIdGenerator(&models.GitlabJob{}) projectIdGen := didgen.NewDomainIdGenerator(&models.GitlabProject{}) pipelineIdGen := didgen.NewDomainIdGenerator(&models.GitlabPipeline{}) - converter, err := api.NewStatefulDataConverter[models.GitlabJob](&api.StatefulDataConverterArgs[models.GitlabJob]{ + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GitlabJob]{ SubtaskCommonArgs: subtaskCommonArgs, Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { clauses := []dal.Clause{ diff --git a/backend/plugins/gitlab/tasks/job_extractor.go b/backend/plugins/gitlab/tasks/job_extractor.go index aec601bd4f1..5827fb68619 100644 --- a/backend/plugins/gitlab/tasks/job_extractor.go +++ b/backend/plugins/gitlab/tasks/job_extractor.go @@ -60,7 +60,7 @@ var ExtractApiJobsMeta = plugin.SubTaskMeta{ func ExtractApiJobs(subtaskCtx plugin.SubTaskContext) errors.Error { subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_JOB_TABLE) - extractor, err := api.NewStatefulApiExtractor[ApiJob](&api.StatefulApiExtractorArgs[ApiJob]{ + extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[ApiJob]{ SubtaskCommonArgs: subtaskCommonArgs, Extract: func(gitlabApiJob *ApiJob, row *api.RawData) ([]interface{}, errors.Error) { // create gitlab commit diff --git a/backend/plugins/gitlab/tasks/mr_assignee_convertor.go b/backend/plugins/gitlab/tasks/mr_assignee_convertor.go index 541634748cf..bf04fbbdeb6 100644 --- a/backend/plugins/gitlab/tasks/mr_assignee_convertor.go +++ b/backend/plugins/gitlab/tasks/mr_assignee_convertor.go @@ -47,7 +47,7 @@ func ConvertMrAssignees(subtaskCtx plugin.SubTaskContext) errors.Error { mrIdGen := didgen.NewDomainIdGenerator(&models.GitlabMergeRequest{}) accountIdGen := didgen.NewDomainIdGenerator(&models.GitlabAccount{}) - converter, err := api.NewStatefulDataConverter[models.GitlabAssignee](&api.StatefulDataConverterArgs[models.GitlabAssignee]{ + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GitlabAssignee]{ SubtaskCommonArgs: subtaskCommonArgs, Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { clauses := []dal.Clause{ diff --git a/backend/plugins/gitlab/tasks/mr_commit_convertor.go b/backend/plugins/gitlab/tasks/mr_commit_convertor.go index 31c78a7fe41..aafeb61933b 100644 --- a/backend/plugins/gitlab/tasks/mr_commit_convertor.go +++ b/backend/plugins/gitlab/tasks/mr_commit_convertor.go @@ -46,7 +46,7 @@ func ConvertApiMergeRequestsCommits(subtaskCtx plugin.SubTaskContext) errors.Err domainIdGenerator := didgen.NewDomainIdGenerator(&models.GitlabMergeRequest{}) - converter, err := api.NewStatefulDataConverter[models.GitlabMrCommit](&api.StatefulDataConverterArgs[models.GitlabMrCommit]{ + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GitlabMrCommit]{ SubtaskCommonArgs: subtaskCommonArgs, Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { clauses := []dal.Clause{ diff --git a/backend/plugins/gitlab/tasks/mr_commit_extractor.go b/backend/plugins/gitlab/tasks/mr_commit_extractor.go index b0568c8c2b6..d101e395b32 100644 --- a/backend/plugins/gitlab/tasks/mr_commit_extractor.go +++ b/backend/plugins/gitlab/tasks/mr_commit_extractor.go @@ -43,7 +43,7 @@ var ExtractApiMrCommitsMeta = plugin.SubTaskMeta{ func ExtractApiMergeRequestsCommits(subtaskCtx plugin.SubTaskContext) errors.Error { subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_MERGE_REQUEST_COMMITS_TABLE) - extractor, err := api.NewStatefulApiExtractor[GitlabApiCommit](&api.StatefulApiExtractorArgs[GitlabApiCommit]{ + extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[GitlabApiCommit]{ SubtaskCommonArgs: subtaskCommonArgs, Extract: func(gitlabApiCommit *GitlabApiCommit, row *api.RawData) ([]interface{}, errors.Error) { // create gitlab commit diff --git a/backend/plugins/gitlab/tasks/mr_convertor.go b/backend/plugins/gitlab/tasks/mr_convertor.go index 22c6b73c2e1..082c3fcad86 100644 --- a/backend/plugins/gitlab/tasks/mr_convertor.go +++ b/backend/plugins/gitlab/tasks/mr_convertor.go @@ -49,7 +49,7 @@ func ConvertApiMergeRequests(subtaskCtx plugin.SubTaskContext) errors.Error { domainRepoIdGenerator := didgen.NewDomainIdGenerator(&models.GitlabProject{}) domainUserIdGen := didgen.NewDomainIdGenerator(&models.GitlabAccount{}) - converter, err := api.NewStatefulDataConverter[models.GitlabMergeRequest](&api.StatefulDataConverterArgs[models.GitlabMergeRequest]{ + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GitlabMergeRequest]{ SubtaskCommonArgs: subtaskCommonArgs, Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { clauses := []dal.Clause{ diff --git a/backend/plugins/gitlab/tasks/mr_detail_extractor.go b/backend/plugins/gitlab/tasks/mr_detail_extractor.go index cf23cb48c14..7f5ec45d065 100644 --- a/backend/plugins/gitlab/tasks/mr_detail_extractor.go +++ b/backend/plugins/gitlab/tasks/mr_detail_extractor.go @@ -64,7 +64,7 @@ func ExtractApiMergeRequestDetails(subtaskCtx plugin.SubTaskContext) errors.Erro "prType": prType, "prComponent": prComponent, } - extractor, err := api.NewStatefulApiExtractor[MergeRequestRes](&api.StatefulApiExtractorArgs[MergeRequestRes]{ + extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[MergeRequestRes]{ SubtaskCommonArgs: subtaskCommonArgs, BeforeExtract: beforeExtractMr(db, data), Extract: func(mr *MergeRequestRes, row *api.RawData) ([]interface{}, errors.Error) { diff --git a/backend/plugins/gitlab/tasks/mr_extractor.go b/backend/plugins/gitlab/tasks/mr_extractor.go index c4647af0866..a76bce7d4d2 100644 --- a/backend/plugins/gitlab/tasks/mr_extractor.go +++ b/backend/plugins/gitlab/tasks/mr_extractor.go @@ -125,7 +125,7 @@ func ExtractApiMergeRequests(subtaskCtx plugin.SubTaskContext) errors.Error { "prComponent": prComponent, } - extractor, err := api.NewStatefulApiExtractor[MergeRequestRes](&api.StatefulApiExtractorArgs[MergeRequestRes]{ + extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[MergeRequestRes]{ SubtaskCommonArgs: subtaskCommonArgs, BeforeExtract: beforeExtractMr(db, data), Extract: func(mr *MergeRequestRes, row *api.RawData) ([]interface{}, errors.Error) { diff --git a/backend/plugins/gitlab/tasks/mr_label_convertor.go b/backend/plugins/gitlab/tasks/mr_label_convertor.go index ccb0a3bc0fd..ca77fe68042 100644 --- a/backend/plugins/gitlab/tasks/mr_label_convertor.go +++ b/backend/plugins/gitlab/tasks/mr_label_convertor.go @@ -46,7 +46,7 @@ func ConvertMrLabels(subtaskCtx plugin.SubTaskContext) errors.Error { mrIdGen := didgen.NewDomainIdGenerator(&models.GitlabMergeRequest{}) - converter, err := api.NewStatefulDataConverter[models.GitlabMrLabel](&api.StatefulDataConverterArgs[models.GitlabMrLabel]{ + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GitlabMrLabel]{ SubtaskCommonArgs: subtaskCommonArgs, Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { clauses := []dal.Clause{ diff --git a/backend/plugins/gitlab/tasks/mr_note_convertor.go b/backend/plugins/gitlab/tasks/mr_note_convertor.go index d1291d464da..a55cc567310 100644 --- a/backend/plugins/gitlab/tasks/mr_note_convertor.go +++ b/backend/plugins/gitlab/tasks/mr_note_convertor.go @@ -49,7 +49,7 @@ func ConvertMergeRequestNote(subtaskCtx plugin.SubTaskContext) errors.Error { prIdGen := didgen.NewDomainIdGenerator(&models.GitlabMergeRequest{}) accountIdGen := didgen.NewDomainIdGenerator(&models.GitlabAccount{}) - converter, err := api.NewStatefulDataConverter[models.GitlabMrComment](&api.StatefulDataConverterArgs[models.GitlabMrComment]{ + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GitlabMrComment]{ SubtaskCommonArgs: subtaskCommonArgs, Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { clauses := []dal.Clause{ diff --git a/backend/plugins/gitlab/tasks/mr_note_extractor.go b/backend/plugins/gitlab/tasks/mr_note_extractor.go index 9354ca86da3..725a2fe484d 100644 --- a/backend/plugins/gitlab/tasks/mr_note_extractor.go +++ b/backend/plugins/gitlab/tasks/mr_note_extractor.go @@ -58,7 +58,7 @@ var ExtractApiMrNotesMeta = plugin.SubTaskMeta{ func ExtractApiMergeRequestsNotes(subtaskCtx plugin.SubTaskContext) errors.Error { subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_MERGE_REQUEST_NOTES_TABLE) - extractor, err := api.NewStatefulApiExtractor[MergeRequestNote](&api.StatefulApiExtractorArgs[MergeRequestNote]{ + extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[MergeRequestNote]{ SubtaskCommonArgs: subtaskCommonArgs, Extract: func(mrNote *MergeRequestNote, row *api.RawData) ([]interface{}, errors.Error) { toolMrNote, err := convertMergeRequestNote(mrNote) diff --git a/backend/plugins/gitlab/tasks/mr_reviewer_convertor.go b/backend/plugins/gitlab/tasks/mr_reviewer_convertor.go index cf5728af858..bac8bb6d729 100644 --- a/backend/plugins/gitlab/tasks/mr_reviewer_convertor.go +++ b/backend/plugins/gitlab/tasks/mr_reviewer_convertor.go @@ -47,7 +47,7 @@ func ConvertMrReviewers(subtaskCtx plugin.SubTaskContext) errors.Error { mrIdGen := didgen.NewDomainIdGenerator(&models.GitlabMergeRequest{}) accountIdGen := didgen.NewDomainIdGenerator(&models.GitlabAccount{}) - converter, err := api.NewStatefulDataConverter[models.GitlabReviewer](&api.StatefulDataConverterArgs[models.GitlabReviewer]{ + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GitlabReviewer]{ SubtaskCommonArgs: subtaskCommonArgs, Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { clauses := []dal.Clause{ diff --git a/backend/plugins/gitlab/tasks/pipeline_commit_convertor.go b/backend/plugins/gitlab/tasks/pipeline_commit_convertor.go index b1cd4e0a45a..2064cc6c921 100644 --- a/backend/plugins/gitlab/tasks/pipeline_commit_convertor.go +++ b/backend/plugins/gitlab/tasks/pipeline_commit_convertor.go @@ -52,7 +52,7 @@ func ConvertPipelineCommits(subtaskCtx plugin.SubTaskContext) errors.Error { pipelineIdGen := didgen.NewDomainIdGenerator(&models.GitlabPipeline{}) - converter, err := api.NewStatefulDataConverter[models.GitlabPipelineProject](&api.StatefulDataConverterArgs[models.GitlabPipelineProject]{ + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GitlabPipelineProject]{ SubtaskCommonArgs: subtaskCommonArgs, Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { clauses := []dal.Clause{ diff --git a/backend/plugins/gitlab/tasks/pipeline_detail_convertor.go b/backend/plugins/gitlab/tasks/pipeline_detail_convertor.go index 6cc8c5358c1..5519461df21 100644 --- a/backend/plugins/gitlab/tasks/pipeline_detail_convertor.go +++ b/backend/plugins/gitlab/tasks/pipeline_detail_convertor.go @@ -52,7 +52,7 @@ func ConvertDetailPipelines(subtaskCtx plugin.SubTaskContext) errors.Error { pipelineIdGen := didgen.NewDomainIdGenerator(&models.GitlabPipeline{}) projectIdGen := didgen.NewDomainIdGenerator(&models.GitlabProject{}) - converter, err := api.NewStatefulDataConverter[models.GitlabPipeline](&api.StatefulDataConverterArgs[models.GitlabPipeline]{ + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GitlabPipeline]{ SubtaskCommonArgs: subtaskCommonArgs, Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { clauses := []dal.Clause{ diff --git a/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go b/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go index ab89cc4adb1..1af785613f5 100644 --- a/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go +++ b/backend/plugins/gitlab/tasks/pipeline_detail_extractor.go @@ -42,7 +42,7 @@ var ExtractApiPipelineDetailsMeta = plugin.SubTaskMeta{ func ExtractApiPipelineDetails(subtaskCtx plugin.SubTaskContext) errors.Error { subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_PIPELINE_DETAILS_TABLE) - extractor, err := api.NewStatefulApiExtractor[ApiPipeline](&api.StatefulApiExtractorArgs[ApiPipeline]{ + extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[ApiPipeline]{ SubtaskCommonArgs: subtaskCommonArgs, Extract: func(gitlabApiPipeline *ApiPipeline, row *api.RawData) ([]interface{}, errors.Error) { gitlabPipeline := &models.GitlabPipeline{ diff --git a/backend/plugins/gitlab/tasks/pipeline_extractor.go b/backend/plugins/gitlab/tasks/pipeline_extractor.go index e5bba1cd6e2..e09e6472bcf 100644 --- a/backend/plugins/gitlab/tasks/pipeline_extractor.go +++ b/backend/plugins/gitlab/tasks/pipeline_extractor.go @@ -70,7 +70,7 @@ var ExtractApiPipelinesMeta = plugin.SubTaskMeta{ func ExtractApiPipelines(subtaskCtx plugin.SubTaskContext) errors.Error { subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_PIPELINE_TABLE) - extractor, err := api.NewStatefulApiExtractor[ApiPipeline](&api.StatefulApiExtractorArgs[ApiPipeline]{ + extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[ApiPipeline]{ SubtaskCommonArgs: subtaskCommonArgs, Extract: func(gitlabApiPipeline *ApiPipeline, row *api.RawData) ([]interface{}, errors.Error) { pipelineProject := &models.GitlabPipelineProject{ diff --git a/backend/plugins/gitlab/tasks/trigger_job_extractor.go b/backend/plugins/gitlab/tasks/trigger_job_extractor.go index 0da01208962..8a1e7eaeb00 100644 --- a/backend/plugins/gitlab/tasks/trigger_job_extractor.go +++ b/backend/plugins/gitlab/tasks/trigger_job_extractor.go @@ -57,7 +57,7 @@ var ExtractApiTriggerJobsMeta = plugin.SubTaskMeta{ func ExtractApiTriggerJobs(subtaskCtx plugin.SubTaskContext) errors.Error { subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_TRIGGER_JOB_TABLE) - extractor, err := api.NewStatefulApiExtractor[ApiTriggerJob](&api.StatefulApiExtractorArgs[ApiTriggerJob]{ + extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[ApiTriggerJob]{ SubtaskCommonArgs: subtaskCommonArgs, Extract: func(gitlabApiTriggerJob *ApiTriggerJob, row *api.RawData) ([]interface{}, errors.Error) { gitlabPipeline, err := convertTriggerJob(gitlabApiTriggerJob, data.Options.ProjectId) diff --git a/backend/plugins/jira/tasks/issue_changelog_convertor.go b/backend/plugins/jira/tasks/issue_changelog_convertor.go index 4dc35e5c987..b27fc966681 100644 --- a/backend/plugins/jira/tasks/issue_changelog_convertor.go +++ b/backend/plugins/jira/tasks/issue_changelog_convertor.go @@ -78,7 +78,7 @@ func ConvertIssueChangelogs(subtaskCtx plugin.SubTaskContext) errors.Error { changelogIdGenerator := didgen.NewDomainIdGenerator(&models.JiraIssueChangelogItems{}) accountIdGen := didgen.NewDomainIdGenerator(&models.JiraAccount{}) - converter, err := api.NewStatefulDataConverter[IssueChangelogItemResult](&api.StatefulDataConverterArgs[IssueChangelogItemResult]{ + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[IssueChangelogItemResult]{ SubtaskCommonArgs: &api.SubtaskCommonArgs{ SubTaskContext: subtaskCtx, Table: RAW_ISSUE_TABLE, diff --git a/backend/plugins/jira/tasks/issue_changelog_extractor.go b/backend/plugins/jira/tasks/issue_changelog_extractor.go index 911185f651f..b5de33a4574 100644 --- a/backend/plugins/jira/tasks/issue_changelog_extractor.go +++ b/backend/plugins/jira/tasks/issue_changelog_extractor.go @@ -47,7 +47,7 @@ func ExtractIssueChangelogs(subtaskCtx plugin.SubTaskContext) errors.Error { if err != nil { return err } - extractor, err := api.NewStatefulApiExtractor[apiv2models.Changelog](&api.StatefulApiExtractorArgs[apiv2models.Changelog]{ + extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[apiv2models.Changelog]{ SubtaskCommonArgs: &api.SubtaskCommonArgs{ SubTaskContext: subtaskCtx, Table: RAW_CHANGELOG_TABLE, diff --git a/backend/plugins/jira/tasks/issue_convertor.go b/backend/plugins/jira/tasks/issue_convertor.go index a55c1642077..fb213a638dd 100644 --- a/backend/plugins/jira/tasks/issue_convertor.go +++ b/backend/plugins/jira/tasks/issue_convertor.go @@ -53,7 +53,7 @@ func ConvertIssues(subtaskCtx plugin.SubTaskContext) errors.Error { boardIdGen := didgen.NewDomainIdGenerator(&models.JiraBoard{}) boardId := boardIdGen.Generate(data.Options.ConnectionId, data.Options.BoardId) - converter, err := api.NewStatefulDataConverter[models.JiraIssue](&api.StatefulDataConverterArgs[models.JiraIssue]{ + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.JiraIssue]{ SubtaskCommonArgs: &api.SubtaskCommonArgs{ SubTaskContext: subtaskCtx, Table: RAW_ISSUE_TABLE, diff --git a/backend/plugins/jira/tasks/issue_extractor.go b/backend/plugins/jira/tasks/issue_extractor.go index 19b89bb7d95..a0ecc6f7045 100644 --- a/backend/plugins/jira/tasks/issue_extractor.go +++ b/backend/plugins/jira/tasks/issue_extractor.go @@ -61,7 +61,7 @@ func ExtractIssues(subtaskCtx plugin.SubTaskContext) errors.Error { if err != nil { return err } - extractor, err := api.NewStatefulApiExtractor[apiv2models.Issue](&api.StatefulApiExtractorArgs[apiv2models.Issue]{ + extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[apiv2models.Issue]{ SubtaskCommonArgs: &api.SubtaskCommonArgs{ SubTaskContext: subtaskCtx, Table: RAW_ISSUE_TABLE, From 81d2a7aa1638b1f0e240de904ecf6b38e73e3ae2 Mon Sep 17 00:00:00 2001 From: Klesh Wong Date: Fri, 6 Sep 2024 15:47:38 +0800 Subject: [PATCH 17/23] fix: shoud not be deleting records not extracted by current extractor --- .../plugins/gitlab/tasks/issue_convertor.go | 4 ++-- backend/plugins/gitlab/tasks/mr_convertor.go | 6 ------ backend/plugins/gitlab/tasks/mr_extractor.go | 21 ------------------- 3 files changed, 2 insertions(+), 29 deletions(-) diff --git a/backend/plugins/gitlab/tasks/issue_convertor.go b/backend/plugins/gitlab/tasks/issue_convertor.go index cefc18c1e59..63af1c6593e 100644 --- a/backend/plugins/gitlab/tasks/issue_convertor.go +++ b/backend/plugins/gitlab/tasks/issue_convertor.go @@ -70,10 +70,10 @@ func ConvertIssues(subtaskCtx plugin.SubTaskContext) errors.Error { }, BeforeConvert: func(issue *models.GitlabIssue, stateManager *api.SubtaskStateManager) errors.Error { issueId := issueIdGen.Generate(data.Options.ConnectionId, issue.GitlabId) - if err := db.Delete(&ticket.IssueAssignee{}, dal.Where("issue_id = ?", issueId)); err != nil { + if err := db.Delete(&ticket.IssueLabel{}, dal.Where("issue_id = ?", issueId)); err != nil { return err } - if err := db.Delete(&ticket.IssueLabel{}, dal.Where("issue_id = ?", issueId)); err != nil { + if err := db.Delete(&ticket.IssueAssignee{}, dal.Where("issue_id = ?", issueId)); err != nil { return err } return nil diff --git a/backend/plugins/gitlab/tasks/mr_convertor.go b/backend/plugins/gitlab/tasks/mr_convertor.go index 082c3fcad86..f2dd24d74a8 100644 --- a/backend/plugins/gitlab/tasks/mr_convertor.go +++ b/backend/plugins/gitlab/tasks/mr_convertor.go @@ -72,12 +72,6 @@ func ConvertApiMergeRequests(subtaskCtx plugin.SubTaskContext) errors.Error { if err := db.Delete(&code.PullRequestReviewer{}, dal.Where("pull_request_id = ?", mrId)); err != nil { return err } - if err := db.Delete(&code.PullRequestComment{}, dal.Where("pull_request_id = ?", mrId)); err != nil { - return err - } - if err := db.Delete(&code.PullRequestCommit{}, dal.Where("pull_request_id = ?", mrId)); err != nil { - return err - } if err := db.Delete(&code.PullRequestLabel{}, dal.Where("pull_request_id = ?", mrId)); err != nil { return err } diff --git a/backend/plugins/gitlab/tasks/mr_extractor.go b/backend/plugins/gitlab/tasks/mr_extractor.go index a76bce7d4d2..3a8457b0cf0 100644 --- a/backend/plugins/gitlab/tasks/mr_extractor.go +++ b/backend/plugins/gitlab/tasks/mr_extractor.go @@ -259,13 +259,6 @@ func beforeExtractMr(db dal.Dal, data *GitlabTaskData) func(mr *MergeRequestRes, if err != nil { return err } - err = db.Delete( - &models.GitlabIssueAssignee{}, - dal.Where("connection_id = ? AND merge_request_id = ?", data.Options.ConnectionId, mr.GitlabId), - ) - if err != nil { - return err - } err = db.Delete( &models.GitlabAssignee{}, dal.Where("connection_id = ? AND merge_request_id = ?", data.Options.ConnectionId, mr.GitlabId), @@ -273,20 +266,6 @@ func beforeExtractMr(db dal.Dal, data *GitlabTaskData) func(mr *MergeRequestRes, if err != nil { return err } - err = db.Delete( - &models.GitlabMrCommit{}, - dal.Where("connection_id = ? AND merge_request_id = ?", data.Options.ConnectionId, mr.GitlabId), - ) - if err != nil { - return err - } - err = db.Delete( - &models.GitlabMrComment{}, - dal.Where("connection_id = ? AND merge_request_id = ?", data.Options.ConnectionId, mr.GitlabId), - ) - if err != nil { - return err - } } return nil } From 967dba89211864b3ff8585fb002bac7f59172dd9 Mon Sep 17 00:00:00 2001 From: Klesh Wong Date: Fri, 6 Sep 2024 16:11:36 +0800 Subject: [PATCH 18/23] fix: jira issue extractor should not deleting sprint_issue --- backend/plugins/jira/tasks/issue_extractor.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/backend/plugins/jira/tasks/issue_extractor.go b/backend/plugins/jira/tasks/issue_extractor.go index a0ecc6f7045..420181d19b9 100644 --- a/backend/plugins/jira/tasks/issue_extractor.go +++ b/backend/plugins/jira/tasks/issue_extractor.go @@ -74,13 +74,6 @@ func ExtractIssues(subtaskCtx plugin.SubTaskContext) errors.Error { BeforeExtract: func(apiIssue *apiv2models.Issue, stateManager *api.SubtaskStateManager) errors.Error { if stateManager.IsIncremental() { err := db.Delete( - &models.JiraSprintIssue{}, - dal.Where("connection_id = ? AND issue_id = ?", data.Options.ConnectionId, apiIssue.ID), - ) - if err != nil { - return err - } - err = db.Delete( &models.JiraIssueLabel{}, dal.Where("connection_id = ? AND issue_id = ?", data.Options.ConnectionId, apiIssue.ID), ) From 5e0dc15ab9283ae5bfa533abba7f1de45a5c0043 Mon Sep 17 00:00:00 2001 From: Klesh Wong Date: Fri, 6 Sep 2024 16:20:47 +0800 Subject: [PATCH 19/23] refactor: remove commit related subtasks --- .../plugins/gitlab/tasks/commit_collector.go | 80 ------------ .../plugins/gitlab/tasks/commit_convertor.go | 108 ---------------- .../plugins/gitlab/tasks/commit_extractor.go | 116 ------------------ .../gitlab/tasks/mr_commit_extractor.go | 41 +++++++ .../gitlab/tasks/mr_detail_extractor.go | 2 +- .../gitlab/tasks/pipeline_detail_convertor.go | 2 +- 6 files changed, 43 insertions(+), 306 deletions(-) delete mode 100644 backend/plugins/gitlab/tasks/commit_collector.go delete mode 100644 backend/plugins/gitlab/tasks/commit_convertor.go delete mode 100644 backend/plugins/gitlab/tasks/commit_extractor.go diff --git a/backend/plugins/gitlab/tasks/commit_collector.go b/backend/plugins/gitlab/tasks/commit_collector.go deleted file mode 100644 index 4bb2e960f8f..00000000000 --- a/backend/plugins/gitlab/tasks/commit_collector.go +++ /dev/null @@ -1,80 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package tasks - -import ( - "github.com/apache/incubator-devlake/core/errors" - "github.com/apache/incubator-devlake/core/models/common" - "github.com/apache/incubator-devlake/core/plugin" - "github.com/apache/incubator-devlake/helpers/pluginhelper/api" -) - -func init() { - RegisterSubtaskMeta(&CollectApiCommitsMeta) -} - -const RAW_COMMIT_TABLE = "gitlab_api_commit" - -var CollectApiCommitsMeta = plugin.SubTaskMeta{ - Name: "Collect Commits", - EntryPoint: CollectApiCommits, - EnabledByDefault: false, - Description: "Collect commit data from gitlab api, does not support either timeFilter or diffSync.", - DomainTypes: []string{plugin.DOMAIN_TYPE_CODE}, - Dependencies: []*plugin.SubTaskMeta{&ConvertJobMeta}, -} - -type GitlabApiCommit struct { - GitlabId string `json:"id"` - Title string - Message string - ProjectId int - ShortId string `json:"short_id"` - AuthorName string `json:"author_name"` - AuthorEmail string `json:"author_email"` - AuthoredDate common.Iso8601Time `json:"authored_date"` - CommitterName string `json:"committer_name"` - CommitterEmail string `json:"committer_email"` - CommittedDate common.Iso8601Time `json:"committed_date"` - WebUrl string `json:"web_url"` - Stats struct { - Additions int - Deletions int - Total int - } -} - -func CollectApiCommits(taskCtx plugin.SubTaskContext) errors.Error { - rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_COMMIT_TABLE) - - collector, err := api.NewApiCollector(api.ApiCollectorArgs{ - RawDataSubTaskArgs: *rawDataSubTaskArgs, - ApiClient: data.ApiClient, - PageSize: 100, - Incremental: false, - UrlTemplate: "projects/{{ .Params.ProjectId }}/repository/commits", - Query: GetQuery, - ResponseParser: GetRawMessageFromResponse, - }) - - if err != nil { - return err - } - - return collector.Execute() -} diff --git a/backend/plugins/gitlab/tasks/commit_convertor.go b/backend/plugins/gitlab/tasks/commit_convertor.go deleted file mode 100644 index c0fb77a794d..00000000000 --- a/backend/plugins/gitlab/tasks/commit_convertor.go +++ /dev/null @@ -1,108 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package tasks - -import ( - "reflect" - - "github.com/apache/incubator-devlake/core/dal" - "github.com/apache/incubator-devlake/core/errors" - "github.com/apache/incubator-devlake/core/models/domainlayer/code" - "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" - "github.com/apache/incubator-devlake/core/plugin" - helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" - "github.com/apache/incubator-devlake/plugins/gitlab/models" -) - -func init() { - RegisterSubtaskMeta(&ConvertCommitsMeta) -} - -var ConvertCommitsMeta = plugin.SubTaskMeta{ - Name: "Convert Commits", - EntryPoint: ConvertApiCommits, - EnabledByDefault: false, - Description: "Update domain layer commit according to GitlabCommit", - DomainTypes: []string{plugin.DOMAIN_TYPE_CODE}, - Dependencies: []*plugin.SubTaskMeta{&ConvertMrLabelsMeta}, -} - -func ConvertApiCommits(taskCtx plugin.SubTaskContext) errors.Error { - rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_COMMIT_TABLE) - db := taskCtx.GetDal() - - // select all commits belongs to the project - clauses := []dal.Clause{ - dal.Select("gc.*"), - dal.From("_tool_gitlab_commits gc"), - dal.Join(`left join _tool_gitlab_project_commits gpc on ( - gpc.commit_sha = gc.sha - )`), - dal.Where("gpc.gitlab_project_id = ? and gpc.connection_id = ? ", - data.Options.ProjectId, data.Options.ConnectionId), - } - cursor, err := db.Cursor(clauses...) - if err != nil { - return err - } - defer cursor.Close() - - // TODO: adopt batch indate operation - //userDidGen := didgen.NewDomainIdGenerator(&models.GitlabAccount{}) - - converter, err := helper.NewDataConverter(helper.DataConverterArgs{ - RawDataSubTaskArgs: *rawDataSubTaskArgs, - InputRowType: reflect.TypeOf(models.GitlabCommit{}), - Input: cursor, - - Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { - gitlabCommit := inputRow.(*models.GitlabCommit) - - // convert commit - commit := &code.Commit{} - commit.Sha = gitlabCommit.Sha - commit.Message = gitlabCommit.Message - commit.Additions = gitlabCommit.Additions - commit.Deletions = gitlabCommit.Deletions - commit.AuthorId = gitlabCommit.AuthorEmail - commit.AuthorName = gitlabCommit.AuthorName - commit.AuthorEmail = gitlabCommit.AuthorEmail - commit.AuthoredDate = gitlabCommit.AuthoredDate - commit.CommitterName = gitlabCommit.CommitterName - commit.CommitterEmail = gitlabCommit.CommitterEmail - commit.CommittedDate = gitlabCommit.CommittedDate - commit.CommitterId = gitlabCommit.CommitterEmail - - // convert repo / commits relationship - repoCommit := &code.RepoCommit{ - RepoId: didgen.NewDomainIdGenerator(&models.GitlabProject{}).Generate(data.Options.ConnectionId, data.Options.ProjectId), - CommitSha: gitlabCommit.Sha, - } - - return []interface{}{ - commit, - repoCommit, - }, nil - }, - }) - if err != nil { - return err - } - - return converter.Execute() -} diff --git a/backend/plugins/gitlab/tasks/commit_extractor.go b/backend/plugins/gitlab/tasks/commit_extractor.go deleted file mode 100644 index 664f8eae3fd..00000000000 --- a/backend/plugins/gitlab/tasks/commit_extractor.go +++ /dev/null @@ -1,116 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package tasks - -import ( - "encoding/json" - - "github.com/apache/incubator-devlake/core/errors" - "github.com/apache/incubator-devlake/core/plugin" - "github.com/apache/incubator-devlake/helpers/pluginhelper/api" - "github.com/apache/incubator-devlake/plugins/gitlab/models" -) - -func init() { - RegisterSubtaskMeta(&ExtractApiCommitsMeta) -} - -var ExtractApiCommitsMeta = plugin.SubTaskMeta{ - Name: "Extract Commits", - EntryPoint: ExtractApiCommits, - EnabledByDefault: false, - Description: "Extract raw commit data into tool layer table GitlabCommit,GitlabAccount and GitlabProjectCommit", - DomainTypes: []string{plugin.DOMAIN_TYPE_CODE}, - Dependencies: []*plugin.SubTaskMeta{&CollectApiCommitsMeta}, -} - -func ExtractApiCommits(taskCtx plugin.SubTaskContext) errors.Error { - rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_COMMIT_TABLE) - - extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ - RawDataSubTaskArgs: *rawDataSubTaskArgs, - Extract: func(row *api.RawData) ([]interface{}, errors.Error) { - // need to extract 3 kinds of entities here - results := make([]interface{}, 0, 3) - - // create gitlab commit - gitlabApiCommit := &GitlabApiCommit{} - err := errors.Convert(json.Unmarshal(row.Data, gitlabApiCommit)) - if err != nil { - return nil, err - } - gitlabCommit, err := ConvertCommit(gitlabApiCommit) - if err != nil { - return nil, err - } - - // create project/commits relationship - gitlabProjectCommit := &models.GitlabProjectCommit{GitlabProjectId: data.Options.ProjectId} - gitlabProjectCommit.CommitSha = gitlabCommit.Sha - - // create gitlab user - GitlabAccountAuthor := &models.GitlabAccount{} - GitlabAccountAuthor.Email = gitlabCommit.AuthorEmail - GitlabAccountAuthor.Name = gitlabCommit.AuthorName - - gitlabProjectCommit.ConnectionId = data.Options.ConnectionId - GitlabAccountAuthor.ConnectionId = data.Options.ConnectionId - results = append(results, gitlabCommit) - results = append(results, gitlabProjectCommit) - results = append(results, GitlabAccountAuthor) - - // For Commiter Email is not same as AuthorEmail - if gitlabCommit.CommitterEmail != GitlabAccountAuthor.Email { - gitlabAccountCommitter := &models.GitlabAccount{} - gitlabAccountCommitter.Email = gitlabCommit.CommitterEmail - gitlabAccountCommitter.Name = gitlabCommit.CommitterName - gitlabAccountCommitter.ConnectionId = data.Options.ConnectionId - results = append(results, gitlabAccountCommitter) - } - - return results, nil - }, - }) - - if err != nil { - return err - } - - return extractor.Execute() -} - -// Convert the API response to our DB model instance -func ConvertCommit(commit *GitlabApiCommit) (*models.GitlabCommit, errors.Error) { - gitlabCommit := &models.GitlabCommit{ - Sha: commit.GitlabId, - Title: commit.Title, - Message: commit.Message, - ShortId: commit.ShortId, - AuthorName: commit.AuthorName, - AuthorEmail: commit.AuthorEmail, - AuthoredDate: commit.AuthoredDate.ToTime(), - CommitterName: commit.CommitterName, - CommitterEmail: commit.CommitterEmail, - CommittedDate: commit.CommittedDate.ToTime(), - WebUrl: commit.WebUrl, - Additions: commit.Stats.Additions, - Deletions: commit.Stats.Deletions, - Total: commit.Stats.Total, - } - return gitlabCommit, nil -} diff --git a/backend/plugins/gitlab/tasks/mr_commit_extractor.go b/backend/plugins/gitlab/tasks/mr_commit_extractor.go index d101e395b32..7a94f9fee55 100644 --- a/backend/plugins/gitlab/tasks/mr_commit_extractor.go +++ b/backend/plugins/gitlab/tasks/mr_commit_extractor.go @@ -40,6 +40,26 @@ var ExtractApiMrCommitsMeta = plugin.SubTaskMeta{ Dependencies: []*plugin.SubTaskMeta{&CollectApiMrCommitsMeta}, } +type GitlabApiCommit struct { + GitlabId string `json:"id"` + Title string + Message string + ProjectId int + ShortId string `json:"short_id"` + AuthorName string `json:"author_name"` + AuthorEmail string `json:"author_email"` + AuthoredDate common.Iso8601Time `json:"authored_date"` + CommitterName string `json:"committer_name"` + CommitterEmail string `json:"committer_email"` + CommittedDate common.Iso8601Time `json:"committed_date"` + WebUrl string `json:"web_url"` + Stats struct { + Additions int + Deletions int + Total int + } +} + func ExtractApiMergeRequestsCommits(subtaskCtx plugin.SubTaskContext) errors.Error { subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_MERGE_REQUEST_COMMITS_TABLE) @@ -86,3 +106,24 @@ func ExtractApiMergeRequestsCommits(subtaskCtx plugin.SubTaskContext) errors.Err return extractor.Execute() } + +// Convert the API response to our DB model instance +func ConvertCommit(commit *GitlabApiCommit) (*models.GitlabCommit, errors.Error) { + gitlabCommit := &models.GitlabCommit{ + Sha: commit.GitlabId, + Title: commit.Title, + Message: commit.Message, + ShortId: commit.ShortId, + AuthorName: commit.AuthorName, + AuthorEmail: commit.AuthorEmail, + AuthoredDate: commit.AuthoredDate.ToTime(), + CommitterName: commit.CommitterName, + CommitterEmail: commit.CommitterEmail, + CommittedDate: commit.CommittedDate.ToTime(), + WebUrl: commit.WebUrl, + Additions: commit.Stats.Additions, + Deletions: commit.Stats.Deletions, + Total: commit.Stats.Total, + } + return gitlabCommit, nil +} diff --git a/backend/plugins/gitlab/tasks/mr_detail_extractor.go b/backend/plugins/gitlab/tasks/mr_detail_extractor.go index 7f5ec45d065..3945752467a 100644 --- a/backend/plugins/gitlab/tasks/mr_detail_extractor.go +++ b/backend/plugins/gitlab/tasks/mr_detail_extractor.go @@ -36,7 +36,7 @@ var ExtractApiMergeRequestDetailsMeta = plugin.SubTaskMeta{ EnabledByDefault: true, Description: "Extract raw merge request Details data into tool layer table GitlabMergeRequest and GitlabReviewer", DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW}, - Dependencies: []*plugin.SubTaskMeta{&ExtractApiCommitsMeta}, + Dependencies: []*plugin.SubTaskMeta{&CollectApiMergeRequestDetailsMeta}, } func ExtractApiMergeRequestDetails(subtaskCtx plugin.SubTaskContext) errors.Error { diff --git a/backend/plugins/gitlab/tasks/pipeline_detail_convertor.go b/backend/plugins/gitlab/tasks/pipeline_detail_convertor.go index 5519461df21..7e36e5079ce 100644 --- a/backend/plugins/gitlab/tasks/pipeline_detail_convertor.go +++ b/backend/plugins/gitlab/tasks/pipeline_detail_convertor.go @@ -41,7 +41,7 @@ var ConvertDetailPipelineMeta = plugin.SubTaskMeta{ EnabledByDefault: true, Description: "Convert tool layer table gitlab_detail_pipeline into domain layer table pipeline", DomainTypes: []string{plugin.DOMAIN_TYPE_CICD}, - Dependencies: []*plugin.SubTaskMeta{&ConvertCommitsMeta}, + Dependencies: []*plugin.SubTaskMeta{&ExtractApiPipelineDetailsMeta}, } func ConvertDetailPipelines(subtaskCtx plugin.SubTaskContext) errors.Error { From 2cb057667df3c79b2495af4107739f9db5e46e5b Mon Sep 17 00:00:00 2001 From: Klesh Wong Date: Mon, 9 Sep 2024 11:33:56 +0800 Subject: [PATCH 20/23] fix: remove commit conversion e2e test --- backend/plugins/gitlab/e2e/mr_commits_test.go | 33 ---------- .../gitlab/e2e/snapshot_tables/commits.csv | 65 ------------------- .../e2e/snapshot_tables/repo_commits.csv | 38 ----------- 3 files changed, 136 deletions(-) delete mode 100644 backend/plugins/gitlab/e2e/snapshot_tables/commits.csv delete mode 100644 backend/plugins/gitlab/e2e/snapshot_tables/repo_commits.csv diff --git a/backend/plugins/gitlab/e2e/mr_commits_test.go b/backend/plugins/gitlab/e2e/mr_commits_test.go index d5ef9d2a43a..2f85cc7756e 100644 --- a/backend/plugins/gitlab/e2e/mr_commits_test.go +++ b/backend/plugins/gitlab/e2e/mr_commits_test.go @@ -133,37 +133,4 @@ func TestGitlabMrCommitDataFlow(t *testing.T) { CSVRelPath: "./snapshot_tables/pull_request_commits.csv", IgnoreTypes: []interface{}{common.Model{}}, }) - - // verify conversion - dataflowTester.FlushTabler(&code.Commit{}) - dataflowTester.FlushTabler(&code.RepoCommit{}) - dataflowTester.Subtask(tasks.ConvertCommitsMeta, taskData) - dataflowTester.VerifyTable( - code.Commit{}, - "./snapshot_tables/commits.csv", - e2ehelper.ColumnWithRawData( - "sha", - "additions", - "deletions", - "dev_eq", - "message", - "author_name", - "author_email", - "authored_date", - "author_id", - "committer_name", - "committer_email", - "committed_date", - "committer_id", - ), - ) - - dataflowTester.VerifyTable( - code.RepoCommit{}, - "./snapshot_tables/repo_commits.csv", - e2ehelper.ColumnWithRawData( - "repo_id", - "commit_sha", - ), - ) } diff --git a/backend/plugins/gitlab/e2e/snapshot_tables/commits.csv b/backend/plugins/gitlab/e2e/snapshot_tables/commits.csv deleted file mode 100644 index ea3ba115e55..00000000000 --- a/backend/plugins/gitlab/e2e/snapshot_tables/commits.csv +++ /dev/null @@ -1,65 +0,0 @@ -sha,additions,deletions,dev_eq,message,author_name,author_email,authored_date,author_id,committer_name,committer_email,committed_date,committer_id,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark -05976c113b02fb71d5c43682bd2f9d96a3454efa,0,0,0,"tab refcatoring -",Nehil Jain,nehil@snaptravel.com,2020-03-24T18:14:03.000+00:00,nehil@snaptravel.com,Nehil Jain,nehil@snaptravel.com,2020-03-24T18:14:03.000+00:00,nehil@snaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,34, -083864201849f6707a34d24e61c43638696f95a6,0,0,0,"Update README to include steps to resolve a potential dbt-utils conflict -",Martin Guindon,mguindon@absolunet.com,2019-08-15T19:32:19.000+00:00,mguindon@absolunet.com,Martin Guindon,mguindon@absolunet.com,2019-08-15T19:32:19.000+00:00,mguindon@absolunet.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,6, -08973065d33c6fd2fb1f592b7a1e0fbe2ee72edd,0,0,0,"add usage day -",Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T18:35:08.000+00:00,tmurphy@gitlab.com,Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T18:35:08.000+00:00,tmurphy@gitlab.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,11, -0a4574d52b5964c37bfcb2952f6c6a8a9c6d5bcf,0,0,0,"fix name -",Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T19:00:50.000+00:00,tmurphy@gitlab.com,Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T19:00:50.000+00:00,tmurphy@gitlab.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,7, -0dbb8bfdbbf7373656fa3eb036bdcbb25ae162da,0,0,0,Apply suggestion to models/base/snowflake_query_history.sql,Nehil,nehil@getsnaptravel.com,2020-03-24T22:39:40.000+00:00,nehil@getsnaptravel.com,Nehil,nehil@getsnaptravel.com,2020-03-24T22:39:40.000+00:00,nehil@getsnaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,26, -1cba71b355d2bd4e1350602cfb9b98cce5a5cb4d,0,0,0,"update documentation -",Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T18:53:44.000+00:00,tmurphy@gitlab.com,Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T18:53:44.000+00:00,tmurphy@gitlab.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,8, -1f16955d45cec904c8968bb27689d91a71e9a2f2,0,0,0,"change cte address -",Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T17:28:44.000+00:00,tmurphy@gitlab.com,Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T17:28:44.000+00:00,tmurphy@gitlab.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,12, -211795deaaaaddb23b797261cdcd148e6083d5b2,0,0,0,Update snowflake_query_history.sql,Nehil,nehil@getsnaptravel.com,2020-03-24T22:44:07.000+00:00,nehil@getsnaptravel.com,Nehil,nehil@getsnaptravel.com,2020-03-24T22:44:07.000+00:00,nehil@getsnaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,21, -25f2b34099a3e1867fcf73a54e50f78f3c8d8f75,0,0,0,"add documentation -",Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T18:47:28.000+00:00,tmurphy@gitlab.com,Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T18:47:28.000+00:00,tmurphy@gitlab.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,9, -382084b42697577d3a6adf71ce73d4b5ddd22977,0,0,0,"add info on dashboard -",emilie,emilie.burke@gmail.com,2019-06-28T10:54:26.000+00:00,emilie.burke@gmail.com,emilie,emilie.burke@gmail.com,2019-06-28T10:54:26.000+00:00,emilie.burke@gmail.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,2, -3f04e0a61d0c4d2dd736a6bcaa3a06826269a533,0,0,0,"finish top level readme -",emilie,emilie.burke@gmail.com,2019-06-28T10:48:49.000+00:00,emilie.burke@gmail.com,emilie,emilie.burke@gmail.com,2019-06-28T10:48:49.000+00:00,emilie.burke@gmail.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,3, -45bed79fcdc6ba3444245f55660f839db50d79c8,0,0,0,Update .gitignore,Nehil,nehil@getsnaptravel.com,2020-03-24T18:15:29.000+00:00,nehil@getsnaptravel.com,Nehil,nehil@getsnaptravel.com,2020-03-24T18:15:29.000+00:00,nehil@getsnaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,33, -48295660f8bd0c0a024849d4407830face57d4bc,0,0,0,"add docs and sources -",emilie,emilie.burke@gmail.com,2020-04-05T14:39:43.000+00:00,emilie.burke@gmail.com,emilie,emilie.burke@gmail.com,2020-04-05T14:39:43.000+00:00,emilie.burke@gmail.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,37, -4f0d2f7b1a47d21b01a68e59c12d6f8885ab230e,0,0,0,"mr comments -",Nehil Jain,nehil@snaptravel.com,2020-03-24T22:40:36.000+00:00,nehil@snaptravel.com,Nehil Jain,nehil@snaptravel.com,2020-03-24T22:40:36.000+00:00,nehil@snaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,24, -553f7613b178301da4075202afe91ebda5d97ab5,0,0,0,Update schema.yml typo for docs,Nehil,nehil@getsnaptravel.com,2020-03-25T19:01:57.000+00:00,nehil@getsnaptravel.com,Nehil,nehil@getsnaptravel.com,2020-03-25T19:01:57.000+00:00,nehil@getsnaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,16, -5a141ad0c28960b02c40e6c476d8df3cc76f58c4,0,0,0,"schema yml, untested -",Nehil Jain,2761999+nehiljain@users.noreply.github.com,2020-03-24T21:18:16.000+00:00,2761999+nehiljain@users.noreply.github.com,Nehil Jain,2761999+nehiljain@users.noreply.github.com,2020-03-24T21:18:16.000+00:00,2761999+nehiljain@users.noreply.github.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,31, -5b6a2be437e0c66ba6276030a61a10583c3f83fa,0,0,0,"merging online changes -",Nehil Jain,nehil@snaptravel.com,2020-03-24T22:42:54.000+00:00,nehil@snaptravel.com,Nehil Jain,nehil@snaptravel.com,2020-03-24T22:42:54.000+00:00,nehil@snaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,22, -5d6c369200c633b06d85c9375f61fef297761ef1,0,0,0,Apply suggestion to models/base/snowflake_query_history.sql,"Taylor A Murphy, PhD",tmurphy@gitlab.com,2020-03-25T13:22:42.000+00:00,tmurphy@gitlab.com,"Taylor A Murphy, PhD",tmurphy@gitlab.com,2020-03-25T13:22:42.000+00:00,tmurphy@gitlab.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,19, -645a724b558a7fa3885fa233d06f68f833d893fc,0,0,0,Apply suggestion to models/base/snowflake_query_history.sql,Nehil,nehil@getsnaptravel.com,2020-03-24T22:39:36.000+00:00,nehil@getsnaptravel.com,Nehil,nehil@getsnaptravel.com,2020-03-24T22:39:36.000+00:00,nehil@getsnaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,27, -64abdcdd9ee9c821f56faf93c5bb0ebc57f575e4,0,0,0,Apply suggestion to models/base/snowflake_query_history.sql,Nehil,nehil@getsnaptravel.com,2020-03-24T22:39:31.000+00:00,nehil@getsnaptravel.com,Nehil,nehil@getsnaptravel.com,2020-03-24T22:39:31.000+00:00,nehil@getsnaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,28, -6ebf519f25328e49a004f5a574c8b69ba7420be1,0,0,0,"add contract rate and dbt-utils -",Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T17:18:15.000+00:00,tmurphy@gitlab.com,Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T17:18:15.000+00:00,tmurphy@gitlab.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,15, -81b12c995b2fc12e20fd05113cf36dbd57b6e7b0,0,0,0,Apply suggestion to models/base/snowflake_query_history.sql,"Taylor A Murphy, PhD",tmurphy@gitlab.com,2020-03-25T13:21:04.000+00:00,tmurphy@gitlab.com,"Taylor A Murphy, PhD",tmurphy@gitlab.com,2020-03-25T13:21:04.000+00:00,tmurphy@gitlab.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,20, -846deac12a1263e4631ced45abbc5e157b23c1be,0,0,0,"testing -",Nehil Jain,nehil@snaptravel.com,2020-03-24T21:27:46.000+00:00,nehil@snaptravel.com,Nehil Jain,nehil@snaptravel.com,2020-03-24T21:27:46.000+00:00,nehil@snaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,30, -8a40765e605966842b05b5d8cdf2dd6890556d09,0,0,0,"change dbt-util version -",Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T17:22:09.000+00:00,tmurphy@gitlab.com,Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T17:22:09.000+00:00,tmurphy@gitlab.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,14, -8a96911c5350f7575215f02f0242f7e27ad3d1f4,0,0,0,"add documentation -",Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T18:46:42.000+00:00,tmurphy@gitlab.com,Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T18:46:42.000+00:00,tmurphy@gitlab.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,10, -91e5666bdfe90e5ada3e65aaa9240b2b0b1a072f,0,0,0,"remove config -",emilie,emilie.burke@gmail.com,2019-08-26T15:34:33.000+00:00,emilie.burke@gmail.com,emilie,emilie.burke@gmail.com,2019-08-26T15:34:33.000+00:00,emilie.burke@gmail.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,17, -93d13cf14a470995ec35186292bf165e1152dc2a,0,0,0,"comments on sql -",Nehil Jain,nehiljain@Hansons-iMac.local,2020-03-24T18:55:11.000+00:00,nehiljain@Hansons-iMac.local,Nehil Jain,nehiljain@Hansons-iMac.local,2020-03-24T18:55:11.000+00:00,nehiljain@Hansons-iMac.local,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,32, -9a80506e158547d87e7ad2902c0196deff5b5c07,0,0,0,"base mmodel for query performance -",Nehil Jain,nehil@snaptravel.com,2020-03-24T12:52:28.000+00:00,nehil@snaptravel.com,Nehil Jain,nehil@snaptravel.com,2020-03-24T12:52:28.000+00:00,nehil@snaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,35, -9e5ec04b0a8273b73936ea150275919b0602c260,0,0,0,"change names -",Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T17:24:35.000+00:00,tmurphy@gitlab.com,Taylor A. Murphy,tmurphy@gitlab.com,2019-08-02T17:24:35.000+00:00,tmurphy@gitlab.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,13, -9ef8183140bae8c39a440d9c7eb47f3bc7dcab75,0,0,0,Apply suggestion to models/base/snowflake_query_history.sql,"Taylor A Murphy, PhD",tmurphy@gitlab.com,2020-03-25T13:22:47.000+00:00,tmurphy@gitlab.com,"Taylor A Murphy, PhD",tmurphy@gitlab.com,2020-03-25T13:22:47.000+00:00,tmurphy@gitlab.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,18, -a09f2d4ae88e78d7424fba1ae6d0c036b677271d,0,0,0,"update readme -",emilie,emilie.burke@gmail.com,2020-04-05T14:43:47.000+00:00,emilie.burke@gmail.com,emilie,emilie.burke@gmail.com,2020-04-05T14:43:47.000+00:00,emilie.burke@gmail.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,36, -abbe0ab2c7bb1dc2cfaa3ef3062f378fb908ba71,0,0,0,"add first bit -",emilie,emilie.burke@gmail.com,2019-06-28T05:40:36.000+00:00,emilie.burke@gmail.com,emilie,emilie.burke@gmail.com,2019-06-28T05:40:36.000+00:00,emilie.burke@gmail.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,5, -ad25fcda15e310a7eb34b3e38ff9b962b54f0e03,0,0,0,"add more info to readme -",emilie,emilie.burke@gmail.com,2019-06-28T11:03:52.000+00:00,emilie.burke@gmail.com,emilie,emilie.burke@gmail.com,2019-06-28T11:03:52.000+00:00,emilie.burke@gmail.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,1, -bb6dfec8dcdacb5b6b37dbd8eb1aa6f9e8c06138,0,0,0,"merging online changes -",Nehil Jain,nehil@snaptravel.com,2020-03-24T22:41:35.000+00:00,nehil@snaptravel.com,Nehil Jain,nehil@snaptravel.com,2020-03-24T22:41:35.000+00:00,nehil@snaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,23, -c4d406c077af582837d3611a46eded70151f93fb,0,0,0,"tested schema and the model -",Nehil Jain,nehil@snaptravel.com,2020-03-24T22:37:23.000+00:00,nehil@snaptravel.com,Nehil Jain,nehil@snaptravel.com,2020-03-24T22:37:23.000+00:00,nehil@snaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,29, -e01d4f03811cd0da9949848731236e0aa261cf54,0,0,0,"move analyses -",emilie,emilie.burke@gmail.com,2019-06-28T10:43:17.000+00:00,emilie.burke@gmail.com,emilie,emilie.burke@gmail.com,2019-06-28T10:43:17.000+00:00,emilie.burke@gmail.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,4, -e24fecacb1a2c85f904bef57ffd5598ca2df023c,0,0,0,Apply suggestion to models/base/snowflake_query_history.sql,Nehil,nehil@getsnaptravel.com,2020-03-24T22:39:42.000+00:00,nehil@getsnaptravel.com,Nehil,nehil@getsnaptravel.com,2020-03-24T22:39:42.000+00:00,nehil@getsnaptravel.com,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,25, diff --git a/backend/plugins/gitlab/e2e/snapshot_tables/repo_commits.csv b/backend/plugins/gitlab/e2e/snapshot_tables/repo_commits.csv deleted file mode 100644 index 1b3ee220d34..00000000000 --- a/backend/plugins/gitlab/e2e/snapshot_tables/repo_commits.csv +++ /dev/null @@ -1,38 +0,0 @@ -repo_id,commit_sha,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark -gitlab:GitlabProject:1:12345678,05976c113b02fb71d5c43682bd2f9d96a3454efa,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,34, -gitlab:GitlabProject:1:12345678,083864201849f6707a34d24e61c43638696f95a6,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,6, -gitlab:GitlabProject:1:12345678,08973065d33c6fd2fb1f592b7a1e0fbe2ee72edd,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,11, -gitlab:GitlabProject:1:12345678,0a4574d52b5964c37bfcb2952f6c6a8a9c6d5bcf,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,7, -gitlab:GitlabProject:1:12345678,0dbb8bfdbbf7373656fa3eb036bdcbb25ae162da,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,26, -gitlab:GitlabProject:1:12345678,1cba71b355d2bd4e1350602cfb9b98cce5a5cb4d,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,8, -gitlab:GitlabProject:1:12345678,1f16955d45cec904c8968bb27689d91a71e9a2f2,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,12, -gitlab:GitlabProject:1:12345678,211795deaaaaddb23b797261cdcd148e6083d5b2,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,21, -gitlab:GitlabProject:1:12345678,25f2b34099a3e1867fcf73a54e50f78f3c8d8f75,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,9, -gitlab:GitlabProject:1:12345678,382084b42697577d3a6adf71ce73d4b5ddd22977,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,2, -gitlab:GitlabProject:1:12345678,3f04e0a61d0c4d2dd736a6bcaa3a06826269a533,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,3, -gitlab:GitlabProject:1:12345678,45bed79fcdc6ba3444245f55660f839db50d79c8,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,33, -gitlab:GitlabProject:1:12345678,48295660f8bd0c0a024849d4407830face57d4bc,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,37, -gitlab:GitlabProject:1:12345678,4f0d2f7b1a47d21b01a68e59c12d6f8885ab230e,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,24, -gitlab:GitlabProject:1:12345678,553f7613b178301da4075202afe91ebda5d97ab5,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,16, -gitlab:GitlabProject:1:12345678,5a141ad0c28960b02c40e6c476d8df3cc76f58c4,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,31, -gitlab:GitlabProject:1:12345678,5b6a2be437e0c66ba6276030a61a10583c3f83fa,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,22, -gitlab:GitlabProject:1:12345678,5d6c369200c633b06d85c9375f61fef297761ef1,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,19, -gitlab:GitlabProject:1:12345678,645a724b558a7fa3885fa233d06f68f833d893fc,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,27, -gitlab:GitlabProject:1:12345678,64abdcdd9ee9c821f56faf93c5bb0ebc57f575e4,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,28, -gitlab:GitlabProject:1:12345678,6ebf519f25328e49a004f5a574c8b69ba7420be1,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,15, -gitlab:GitlabProject:1:12345678,81b12c995b2fc12e20fd05113cf36dbd57b6e7b0,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,20, -gitlab:GitlabProject:1:12345678,846deac12a1263e4631ced45abbc5e157b23c1be,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,30, -gitlab:GitlabProject:1:12345678,8a40765e605966842b05b5d8cdf2dd6890556d09,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,14, -gitlab:GitlabProject:1:12345678,8a96911c5350f7575215f02f0242f7e27ad3d1f4,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,10, -gitlab:GitlabProject:1:12345678,91e5666bdfe90e5ada3e65aaa9240b2b0b1a072f,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,17, -gitlab:GitlabProject:1:12345678,93d13cf14a470995ec35186292bf165e1152dc2a,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,32, -gitlab:GitlabProject:1:12345678,9a80506e158547d87e7ad2902c0196deff5b5c07,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,35, -gitlab:GitlabProject:1:12345678,9e5ec04b0a8273b73936ea150275919b0602c260,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,13, -gitlab:GitlabProject:1:12345678,9ef8183140bae8c39a440d9c7eb47f3bc7dcab75,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,18, -gitlab:GitlabProject:1:12345678,a09f2d4ae88e78d7424fba1ae6d0c036b677271d,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,36, -gitlab:GitlabProject:1:12345678,abbe0ab2c7bb1dc2cfaa3ef3062f378fb908ba71,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,5, -gitlab:GitlabProject:1:12345678,ad25fcda15e310a7eb34b3e38ff9b962b54f0e03,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,1, -gitlab:GitlabProject:1:12345678,bb6dfec8dcdacb5b6b37dbd8eb1aa6f9e8c06138,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,23, -gitlab:GitlabProject:1:12345678,c4d406c077af582837d3611a46eded70151f93fb,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,29, -gitlab:GitlabProject:1:12345678,e01d4f03811cd0da9949848731236e0aa261cf54,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,4, -gitlab:GitlabProject:1:12345678,e24fecacb1a2c85f904bef57ffd5598ca2df023c,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,25, From 4eabe806a8ac3d7bb857242ed7d3b00e8d92e94e Mon Sep 17 00:00:00 2001 From: Lynwee <1507509064@qq.com> Date: Mon, 9 Sep 2024 12:59:43 +0800 Subject: [PATCH 21/23] feat(gitlab): update ExtractApiChildPipelines (#8016) --- .../gitlab/tasks/child_pipeline_extractor.go | 29 +++++-------------- 1 file changed, 7 insertions(+), 22 deletions(-) diff --git a/backend/plugins/gitlab/tasks/child_pipeline_extractor.go b/backend/plugins/gitlab/tasks/child_pipeline_extractor.go index 4c7d8d8755e..62c5948fc7d 100644 --- a/backend/plugins/gitlab/tasks/child_pipeline_extractor.go +++ b/backend/plugins/gitlab/tasks/child_pipeline_extractor.go @@ -18,8 +18,6 @@ limitations under the License. package tasks import ( - "encoding/json" - "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/common" "github.com/apache/incubator-devlake/core/plugin" @@ -40,32 +38,19 @@ var ExtractApiChildPipelinesMeta = plugin.SubTaskMeta{ Dependencies: []*plugin.SubTaskMeta{&CollectApiChildPipelinesMeta}, } -func ExtractApiChildPipelines(taskCtx plugin.SubTaskContext) errors.Error { - rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_CHILD_PIPELINE_TABLE) - - extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ - RawDataSubTaskArgs: *rawDataSubTaskArgs, - Extract: func(row *api.RawData) ([]interface{}, errors.Error) { - gitlabApiChildPipeline := &ApiPipeline{} - err := errors.Convert(json.Unmarshal(row.Data, gitlabApiChildPipeline)) - if err != nil { - return nil, err - } - pipelineProject := convertApiPipelineToGitlabPipelineProject(gitlabApiChildPipeline, data.Options.ConnectionId, data.Options.ProjectId) +func ExtractApiChildPipelines(subtaskCtx plugin.SubTaskContext) errors.Error { + subtaskCommonArgs, data := CreateSubtaskCommonArgs(subtaskCtx, RAW_CHILD_PIPELINE_TABLE) + extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[ApiPipeline]{ + SubtaskCommonArgs: subtaskCommonArgs, + Extract: func(gitlabApiPipeline *ApiPipeline, row *api.RawData) ([]interface{}, errors.Error) { + pipelineProject := convertApiPipelineToGitlabPipelineProject(gitlabApiPipeline, data.Options.ConnectionId, data.Options.ProjectId) return []interface{}{pipelineProject}, nil }, }) - - if err != nil { - return err - } - - err = extractor.Execute() if err != nil { return err } - - return nil + return extractor.Execute() } func convertApiPipelineToGitlabPipelineProject(gitlabApiChildPipeline *ApiPipeline, connectionId uint64, projectId int) *models.GitlabPipelineProject { From 3c6d9fdeedc36f8b54f75f16baf3bf4ac9d95a7a Mon Sep 17 00:00:00 2001 From: Klesh Wong Date: Tue, 10 Sep 2024 15:02:23 +0800 Subject: [PATCH 22/23] fix: retransform should run in fullsync mode --- backend/server/api/blueprints/blueprints.go | 1 + config-ui/src/routes/blueprint/detail/status-panel.tsx | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/server/api/blueprints/blueprints.go b/backend/server/api/blueprints/blueprints.go index 23383d4682f..841027424cc 100644 --- a/backend/server/api/blueprints/blueprints.go +++ b/backend/server/api/blueprints/blueprints.go @@ -166,6 +166,7 @@ func Patch(c *gin.Context) { // @Accept application/json // @Param blueprintId path string true "blueprintId" // @Param skipCollectors body models.TriggerSyncPolicy false "json" +// @Param fullSync body models.TriggerSyncPolicy false "json" // @Success 200 {object} models.Pipeline // @Failure 400 {object} shared.ApiBody "Bad Request" // @Failure 500 {object} shared.ApiBody "Internal Error" diff --git a/config-ui/src/routes/blueprint/detail/status-panel.tsx b/config-ui/src/routes/blueprint/detail/status-panel.tsx index a4feee6dfc3..0dbf72a9b65 100644 --- a/config-ui/src/routes/blueprint/detail/status-panel.tsx +++ b/config-ui/src/routes/blueprint/detail/status-panel.tsx @@ -134,7 +134,7 @@ export const StatusPanel = ({ from, blueprint, pipelineId, onRefresh }: Props) = type="primary" disabled={!blueprint.enable} loading={operating} - onClick={() => handleRun({ skipCollectors: true })} + onClick={() => handleRun({ skipCollectors: true, fullSync: true })} > Re-transform Data From 6c144fae91fde4aba225f08c03a47f4ef0628fe2 Mon Sep 17 00:00:00 2001 From: Klesh Wong Date: Thu, 12 Sep 2024 09:45:49 +0800 Subject: [PATCH 23/23] fix: gitlab issue assignees are not being converted --- backend/plugins/gitlab/tasks/issue_assignee_convertor.go | 2 +- backend/plugins/gitlab/tasks/mr_assignee_convertor.go | 2 +- backend/plugins/gitlab/tasks/mr_commit_convertor.go | 2 +- backend/plugins/gitlab/tasks/mr_label_convertor.go | 2 +- backend/plugins/gitlab/tasks/mr_reviewer_convertor.go | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/backend/plugins/gitlab/tasks/issue_assignee_convertor.go b/backend/plugins/gitlab/tasks/issue_assignee_convertor.go index 2f063609ff2..ce5ce593fcc 100644 --- a/backend/plugins/gitlab/tasks/issue_assignee_convertor.go +++ b/backend/plugins/gitlab/tasks/issue_assignee_convertor.go @@ -38,7 +38,7 @@ var ConvertIssueAssigneeMeta = plugin.SubTaskMeta{ Description: "Convert tool layer table _tool_gitlab_issue_assignees into domain layer table issue_assignees", DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, DependencyTables: []string{models.GitlabIssueAssignee{}.TableName()}, - Dependencies: []*plugin.SubTaskMeta{&ExtractApiIssuesMeta}, + Dependencies: []*plugin.SubTaskMeta{&ConvertIssuesMeta}, } func ConvertIssueAssignee(subtaskCtx plugin.SubTaskContext) errors.Error { diff --git a/backend/plugins/gitlab/tasks/mr_assignee_convertor.go b/backend/plugins/gitlab/tasks/mr_assignee_convertor.go index bf04fbbdeb6..d22714a7485 100644 --- a/backend/plugins/gitlab/tasks/mr_assignee_convertor.go +++ b/backend/plugins/gitlab/tasks/mr_assignee_convertor.go @@ -37,7 +37,7 @@ var ConvertMrAssigneesMeta = plugin.SubTaskMeta{ EnabledByDefault: true, Description: "Convert tool layer table gitlab_assignees into domain layer table pull_request_assignees", DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW}, - Dependencies: []*plugin.SubTaskMeta{&ExtractApiMergeRequestDetailsMeta}, + Dependencies: []*plugin.SubTaskMeta{&ConvertApiMergeRequestsMeta}, } func ConvertMrAssignees(subtaskCtx plugin.SubTaskContext) errors.Error { diff --git a/backend/plugins/gitlab/tasks/mr_commit_convertor.go b/backend/plugins/gitlab/tasks/mr_commit_convertor.go index aafeb61933b..6f7ff2a5f8e 100644 --- a/backend/plugins/gitlab/tasks/mr_commit_convertor.go +++ b/backend/plugins/gitlab/tasks/mr_commit_convertor.go @@ -37,7 +37,7 @@ var ConvertApiMrCommitsMeta = plugin.SubTaskMeta{ EnabledByDefault: true, Description: "Add domain layer PullRequestCommit according to GitlabMrCommit", DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW}, - Dependencies: []*plugin.SubTaskMeta{&ConvertMrCommentMeta}, + Dependencies: []*plugin.SubTaskMeta{&ConvertApiMergeRequestsMeta}, } func ConvertApiMergeRequestsCommits(subtaskCtx plugin.SubTaskContext) errors.Error { diff --git a/backend/plugins/gitlab/tasks/mr_label_convertor.go b/backend/plugins/gitlab/tasks/mr_label_convertor.go index ca77fe68042..b5f29a1a84f 100644 --- a/backend/plugins/gitlab/tasks/mr_label_convertor.go +++ b/backend/plugins/gitlab/tasks/mr_label_convertor.go @@ -37,7 +37,7 @@ var ConvertMrLabelsMeta = plugin.SubTaskMeta{ EnabledByDefault: true, Description: "Convert tool layer table gitlab_mr_labels into domain layer table pull_request_labels", DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW}, - Dependencies: []*plugin.SubTaskMeta{&ConvertIssueLabelsMeta}, + Dependencies: []*plugin.SubTaskMeta{&ConvertApiMergeRequestsMeta}, } func ConvertMrLabels(subtaskCtx plugin.SubTaskContext) errors.Error { diff --git a/backend/plugins/gitlab/tasks/mr_reviewer_convertor.go b/backend/plugins/gitlab/tasks/mr_reviewer_convertor.go index bac8bb6d729..f9298b093f6 100644 --- a/backend/plugins/gitlab/tasks/mr_reviewer_convertor.go +++ b/backend/plugins/gitlab/tasks/mr_reviewer_convertor.go @@ -37,7 +37,7 @@ var ConvertMrReviewersMeta = plugin.SubTaskMeta{ EnabledByDefault: true, Description: "Convert tool layer table gitlab_reviewers into domain layer table pull_request_reviewers", DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW}, - Dependencies: []*plugin.SubTaskMeta{&ExtractApiMergeRequestDetailsMeta}, + Dependencies: []*plugin.SubTaskMeta{&ConvertApiMergeRequestsMeta}, } func ConvertMrReviewers(subtaskCtx plugin.SubTaskContext) errors.Error {